diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8ee03c3d999c4..c591459f01bb7 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -100,6 +100,12 @@ JDK 10 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
 pointing to the Java home of a JDK 8 installation. Note that this mechanism can
 be used to test against other JDKs as well, this is not only limited to JDK 8.
 
+> Note: It is also required to have `JAVA7_HOME`, `JAVA8_HOME` and
+`JAVA10_HOME` available so that the tests can pass.
+
+> Warning: do not use `sdkman` for Java installations which do not have proper
+`jrunscript` for jdk distributions.
+
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
 using the wrapper via the `gradlew` script in the root of the repository.
 
diff --git a/README.textile b/README.textile
index ce7b3b7d34476..aa61c03574275 100644
--- a/README.textile
+++ b/README.textile
@@ -209,10 +209,6 @@ The distribution for each project will be created under the @build/distributions
 
 See the "TESTING":TESTING.asciidoc file for more information about running the Elasticsearch test suite.
 
-h3. Upgrading from Elasticsearch 1.x?
+h3. Upgrading from older Elasticsearch versions
 
-In order to ensure a smooth upgrade process from earlier versions of
-Elasticsearch (1.x), it is required to perform a full cluster restart. Please
-see the "setup reference":
-https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html
-for more details on the upgrade process.
+In order to ensure a smooth upgrade process from earlier versions of Elasticsearch, please see our "upgrade documentation":https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html for more details on the upgrade process.
diff --git a/Vagrantfile b/Vagrantfile
index de344e1818360..7322399fed576 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -115,11 +115,6 @@ Vagrant.configure(2) do |config|
   'opensuse-42'.tap do |box|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/opensuse-42-x86_64'
-
-      # https://github.com/elastic/elasticsearch/issues/30295
-      config.vm.provider 'virtualbox' do |vbox|
-        vbox.customize ['storagectl', :id, '--name', 'SATA Controller', '--hostiocache', 'on']
-      end
       suse_common config, box
     end
   end
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 03aaac7f3c4e6..5be14bab0ecbc 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -4,36 +4,39 @@ This directory contains the microbenchmark suite of Elasticsearch. It relies on
 
 ## Purpose
 
-We do not want to microbenchmark everything but the kitchen sink and should typically rely on our 
-[macrobenchmarks](https://elasticsearch-benchmarks.elastic.co/app/kibana#/dashboard/Nightly-Benchmark-Overview) with 
-[Rally](http://github.com/elastic/rally). Microbenchmarks are intended to spot performance regressions in performance-critical components. 
+We do not want to microbenchmark everything but the kitchen sink and should typically rely on our
+[macrobenchmarks](https://elasticsearch-benchmarks.elastic.co/app/kibana#/dashboard/Nightly-Benchmark-Overview) with
+[Rally](http://github.com/elastic/rally). Microbenchmarks are intended to spot performance regressions in performance-critical components.
 The microbenchmark suite is also handy for ad-hoc microbenchmarks but please remove them again before merging your PR.
 
 ## Getting Started
 
-Just run `gradle :benchmarks:jmh` from the project root directory. It will build all microbenchmarks, execute them and print the result.
+Just run `gradlew -p benchmarks run` from the project root
+directory. It will build all microbenchmarks, execute them and print
+the result.
 
 ## Running Microbenchmarks
 
-Benchmarks are always run via Gradle with `gradle :benchmarks:jmh`.
- 
-Running via an IDE is not supported as the results are meaningless (we have no control over the JVM running the benchmarks).
+Running via an IDE is not supported as the results are meaningless
+because we have no control over the JVM running the benchmarks.
 
-If you want to run a specific benchmark class, e.g. `org.elasticsearch.benchmark.MySampleBenchmark` or have special requirements 
-generate the uberjar with `gradle :benchmarks:jmhJar` and run it directly with:
+If you want to run a specific benchmark class like, say,
+`MemoryStatsBenchmark`, you can use `--args`:
 
 ```
-java -jar benchmarks/build/distributions/elasticsearch-benchmarks-*.jar
+gradlew -p benchmarks run --args ' MemoryStatsBenchmark'
 ```
 
-JMH supports lots of command line parameters. Add `-h` to the command above to see the available command line options.
+Everything in the `'` gets sent on the command line to JMH. The leading ` `
+inside the `'`s is important. Without it parameters are sometimes sent to
+gradle.
 
 ## Adding Microbenchmarks
 
-Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the 
+Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the
 [JMH samples](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/).
 
-In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and 
+In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and
 end the class name of a benchmark with `Benchmark`. To have JMH execute a benchmark, annotate the respective methods with `@Benchmark`.
 
 ## Tips and Best Practices
@@ -42,15 +45,15 @@ To get realistic results, you should exercise care when running benchmarks. Here
 
 ### Do
 
-* Ensure that the system executing your microbenchmarks has as little load as possible. Shutdown every process that can cause unnecessary 
+* Ensure that the system executing your microbenchmarks has as little load as possible. Shutdown every process that can cause unnecessary
   runtime jitter. Watch the `Error` column in the benchmark results to see the run-to-run variance.
 * Ensure to run enough warmup iterations to get the benchmark into a stable state. If you are unsure, don't change the defaults.
 * Avoid CPU migrations by pinning your benchmarks to specific CPU cores. On Linux you can use `taskset`.
-* Fix the CPU frequency to avoid Turbo Boost from kicking in and skewing your results. On Linux you can use `cpufreq-set` and the 
+* Fix the CPU frequency to avoid Turbo Boost from kicking in and skewing your results. On Linux you can use `cpufreq-set` and the
   `performance` CPU governor.
 * Vary the problem input size with `@Param`.
 * Use the integrated profilers in JMH to dig deeper if benchmark results to not match your hypotheses:
-    * Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews 
+    * Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews
    your results. If so, try to force a GC between runs (`-gc true`) but watch out for the caveats.
     * Use `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots.
 * Have your benchmarks peer-reviewed.
@@ -59,4 +62,4 @@ To get realistic results, you should exercise care when running benchmarks. Here
 
 * Blindly believe the numbers that your microbenchmark produces but verify them by measuring e.g. with `-prof perfasm`.
 * Run more threads than your number of CPU cores (in case you run multi-threaded microbenchmarks).
-* Look only at the `Score` column and ignore `Error`. Instead take countermeasures to keep `Error` low / variance explainable.
\ No newline at end of file
+* Look only at the `Score` column and ignore `Error`. Instead take countermeasures to keep `Error` low / variance explainable.
diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
index 80d1982300dd1..0838af7287126 100644
--- a/benchmarks/build.gradle
+++ b/benchmarks/build.gradle
@@ -18,11 +18,8 @@
  */
 
 apply plugin: 'elasticsearch.build'
-
-// order of this section matters, see: https://github.com/johnrengelman/shadow/issues/336
-apply plugin: 'application' // have the shadow plugin provide the runShadow task
+apply plugin: 'application'
 mainClassName = 'org.openjdk.jmh.Main'
-apply plugin: 'com.github.johnrengelman.shadow' // build an uberjar with all benchmarks
 
 // Not published so no need to assemble
 tasks.remove(assemble)
@@ -50,10 +47,8 @@ compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-u
 // needs to be added separately otherwise Gradle will quote it and javac will fail
 compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])
 
-forbiddenApis {
-    // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
-    ignoreFailures = true
-}
+// classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
+forbiddenApisMain.enabled = false
 
 // No licenses for our benchmark deps (we don't ship benchmarks)
 dependencyLicenses.enabled = false
@@ -69,20 +64,3 @@ thirdPartyAudit.excludes = [
         'org.openjdk.jmh.profile.HotspotRuntimeProfiler',
         'org.openjdk.jmh.util.Utils'
 ]
-
-runShadow {
-    executable = new File(project.runtimeJavaHome, 'bin/java')
-}
-
-// alias the shadowJar and runShadow tasks to abstract from the concrete plugin that we are using and provide a more consistent interface
-task jmhJar(
-        dependsOn: shadowJar,
-        description: 'Generates an uberjar with the microbenchmarks and all dependencies',
-        group: 'Benchmark'
-)
-
-task jmh(
-        dependsOn: runShadow,
-        description: 'Runs all microbenchmarks',
-        group: 'Benchmark'
-)
diff --git a/benchmarks/src/main/resources/log4j2.properties b/benchmarks/src/main/resources/log4j2.properties
index c3ae1fe56d3d1..808d611670f31 100644
--- a/benchmarks/src/main/resources/log4j2.properties
+++ b/benchmarks/src/main/resources/log4j2.properties
@@ -1,7 +1,7 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %m%n
 
 # Do not log at all if it is not really critical - we're in a benchmark
 rootLogger.level = error
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 5a962a5138b5f..306a2bcb58bd1 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -87,6 +87,8 @@ class BuildPlugin implements Plugin<Project> {
         project.pluginManager.apply('nebula.info-scm')
         project.pluginManager.apply('nebula.info-jar')
 
+        project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
+
         globalBuildInfo(project)
         configureRepositories(project)
         configureConfigurations(project)
@@ -101,6 +103,7 @@ class BuildPlugin implements Plugin<Project> {
         configureDependenciesInfo(project)
     }
 
+
     /** Performs checks on the build environment and prints information about the build environment. */
     static void globalBuildInfo(Project project) {
         if (project.rootProject.ext.has('buildChecksDone') == false) {
@@ -116,12 +119,14 @@ class BuildPlugin implements Plugin<Project> {
 
             final Map<Integer, String> javaVersions = [:]
             for (int version = 7; version <= Integer.parseInt(minimumCompilerVersion.majorVersion); version++) {
-                javaVersions.put(version, findJavaHome(version));
+                if(System.getenv(getJavaHomeEnvVarName(version.toString())) != null) {
+                    javaVersions.put(version, findJavaHome(version.toString()));
+                }
             }
 
             String javaVendor = System.getProperty('java.vendor')
-            String javaVersion = System.getProperty('java.version')
-            String gradleJavaVersionDetails = "${javaVendor} ${javaVersion}" +
+            String gradleJavaVersion = System.getProperty('java.version')
+            String gradleJavaVersionDetails = "${javaVendor} ${gradleJavaVersion}" +
                 " [${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]"
 
             String compilerJavaVersionDetails = gradleJavaVersionDetails
@@ -144,33 +149,33 @@ class BuildPlugin implements Plugin<Project> {
             // Build debugging info
             println '======================================='
             println 'Elasticsearch Build Hamster says Hello!'
-            println '======================================='
             println "  Gradle Version        : ${project.gradle.gradleVersion}"
             println "  OS Info               : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})"
             if (gradleJavaVersionDetails != compilerJavaVersionDetails || gradleJavaVersionDetails != runtimeJavaVersionDetails) {
-                println "  JDK Version (gradle)  : ${gradleJavaVersionDetails}"
-                println "  JAVA_HOME (gradle)    : ${gradleJavaHome}"
-                println "  JDK Version (compile) : ${compilerJavaVersionDetails}"
-                println "  JAVA_HOME (compile)   : ${compilerJavaHome}"
-                println "  JDK Version (runtime) : ${runtimeJavaVersionDetails}"
-                println "  JAVA_HOME (runtime)   : ${runtimeJavaHome}"
+                println "  Compiler JDK Version  : ${getPaddedMajorVersion(compilerJavaVersionEnum)} (${compilerJavaVersionDetails})"
+                println "  Compiler java.home    : ${compilerJavaHome}"
+                println "  Runtime JDK Version   : ${getPaddedMajorVersion(runtimeJavaVersionEnum)} (${runtimeJavaVersionDetails})"
+                println "  Runtime java.home     : ${runtimeJavaHome}"
+                println "  Gradle JDK Version    : ${getPaddedMajorVersion(JavaVersion.toVersion(gradleJavaVersion))} (${gradleJavaVersionDetails})"
+                println "  Gradle java.home      : ${gradleJavaHome}"
             } else {
-                println "  JDK Version           : ${gradleJavaVersionDetails}"
+                println "  JDK Version           : ${getPaddedMajorVersion(JavaVersion.toVersion(gradleJavaVersion))} (${gradleJavaVersionDetails})"
                 println "  JAVA_HOME             : ${gradleJavaHome}"
             }
             println "  Random Testing Seed   : ${project.testSeed}"
+            println '======================================='
 
             // enforce Java version
             if (compilerJavaVersionEnum < minimumCompilerVersion) {
                 final String message =
-                        "the environment variable JAVA_HOME must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
+                        "the compiler java.home must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
                                 " but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]"
                 throw new GradleException(message)
             }
 
             if (runtimeJavaVersionEnum < minimumRuntimeVersion) {
                 final String message =
-                        "the environment variable RUNTIME_JAVA_HOME must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
+                        "the runtime java.home must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
                                 " but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]"
                 throw new GradleException(message)
             }
@@ -205,6 +210,7 @@ class BuildPlugin implements Plugin<Project> {
             project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
             project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
             project.rootProject.ext.inFipsJvm = inFipsJvm
+            project.rootProject.ext.gradleJavaVersion = JavaVersion.toVersion(gradleJavaVersion)
         }
 
         project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
@@ -217,11 +223,20 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
         project.ext.javaVersions = project.rootProject.ext.javaVersions
         project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm
+        project.ext.gradleJavaVersion = project.rootProject.ext.gradleJavaVersion
+    }
+
+    private static String getPaddedMajorVersion(JavaVersion compilerJavaVersionEnum) {
+        compilerJavaVersionEnum.getMajorVersion().toString().padLeft(2)
     }
 
     private static String findCompilerJavaHome() {
-        final String javaHome = System.getenv('JAVA_HOME')
-        if (javaHome == null) {
+        final String compilerJavaHome = System.getenv('JAVA_HOME')
+        final String compilerJavaProperty = System.getProperty('compiler.java')
+        if (compilerJavaProperty != null) {
+            compilerJavaHome = findJavaHome(compilerJavaProperty)
+        }
+        if (compilerJavaHome == null) {
             if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
                 // IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
                 return Jvm.current().javaHome
@@ -233,11 +248,24 @@ class BuildPlugin implements Plugin<Project> {
                 )
             }
         }
-        return javaHome
+        return compilerJavaHome
     }
 
-    private static String findJavaHome(int version) {
-        return System.getenv('JAVA' + version + '_HOME')
+    private static String findJavaHome(String version) {
+        String versionedVarName = getJavaHomeEnvVarName(version)
+        String versionedJavaHome = System.getenv(versionedVarName);
+        if (versionedJavaHome == null) {
+            throw new GradleException(
+                    "$versionedVarName must be set to build Elasticsearch. " +
+                            "Note that if the variable was just set you might have to run `./gradlew --stop` for " +
+                            "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details."
+            )
+        }
+        return versionedJavaHome
+    }
+
+    private static String getJavaHomeEnvVarName(String version) {
+        return 'JAVA' + version + '_HOME'
     }
 
     /** Add a check before gradle execution phase which ensures java home for the given java version is set. */
@@ -271,7 +299,10 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     private static String findRuntimeJavaHome(final String compilerJavaHome) {
-        assert compilerJavaHome != null
+        String runtimeJavaProperty = System.getProperty("runtime.java")
+        if (runtimeJavaProperty != null) {
+            return findJavaHome(runtimeJavaProperty)
+        }
         return System.getenv('RUNTIME_JAVA_HOME') ?: compilerJavaHome
     }
 
@@ -405,6 +436,10 @@ class BuildPlugin implements Plugin<Project> {
             repos.mavenLocal()
         }
         repos.mavenCentral()
+        repos.maven {
+            name "elastic"
+            url "https://artifacts.elastic.co/maven"
+        }
         String luceneVersion = VersionProperties.lucene
         if (luceneVersion.contains('-snapshot')) {
             // extract the revision number from the version with a regex matcher
@@ -764,6 +799,12 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.task', path
             systemProperty 'tests.security.manager', 'true'
             systemProperty 'jna.nosys', 'true'
+            systemProperty 'compiler.java', project.ext.compilerJavaVersion.getMajorVersion()
+            if (project.ext.inFipsJvm) {
+                systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion() + "FIPS"
+            } else {
+                systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion()
+            }
             // TODO: remove setting logging level via system property
             systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {
@@ -777,11 +818,19 @@ class BuildPlugin implements Plugin<Project> {
                     systemProperty property.getKey(), property.getValue()
                 }
             }
+
+            // TODO: remove this once joda time is removed from scripting in 7.0
+            systemProperty 'es.scripting.use_java_time', 'true'
+
+            // TODO: remove this once ctx isn't added to update script params in 7.0
+            systemProperty 'es.scripting.update.ctx_in_params', 'false'
+
             // Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM
             if (project.inFipsJvm) {
                 systemProperty 'javax.net.ssl.trustStorePassword', 'password'
                 systemProperty 'javax.net.ssl.keyStorePassword', 'password'
             }
+
             boolean assertionsEnabled = Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))
             enableSystemAssertions assertionsEnabled
             enableAssertions assertionsEnabled
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
index ec012633f0893..8c0eedeb6f547 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
@@ -284,6 +284,10 @@ public class SnippetsTask extends DefaultTask {
                     contents.append(line).append('\n')
                     return
                 }
+                // Allow line continuations for console snippets within lists
+                if (snippet != null && line.trim() == '+') {
+                    return
+                }
                 // Just finished
                 emit()
             }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
index 3709805680d7a..598d82d5d9aa7 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -20,10 +20,14 @@ package org.elasticsearch.gradle.precommit
 
 import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
+import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.file.FileCollection
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.plugins.quality.Checkstyle
+import org.gradle.api.tasks.JavaExec
+import org.gradle.api.tasks.StopExecutionException
 
 /**
  * Validation tasks which should be run before committing. These run before tests.
@@ -40,7 +44,11 @@ class PrecommitTasks {
             project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
             project.tasks.create('filepermissions', FilePermissionsTask.class),
             project.tasks.create('jarHell', JarHellTask.class),
-            project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)]
+            project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)
+        ]
+
+        // Configure it but don't add it as a dependency yet
+        configureForbiddenApisCli(project)
 
         // tasks with just tests don't need dependency licenses, so this flag makes adding
         // the task optional
@@ -96,9 +104,58 @@ class PrecommitTasks {
         }
         Task forbiddenApis = project.tasks.findByName('forbiddenApis')
         forbiddenApis.group = "" // clear group, so this does not show up under verification tasks
+
         return forbiddenApis
     }
 
+    private static Task configureForbiddenApisCli(Project project) {
+        project.configurations.create("forbiddenApisCliJar")
+        project.dependencies {
+            forbiddenApisCliJar 'de.thetaphi:forbiddenapis:2.5'
+        }
+        Task forbiddenApisCli = project.tasks.create('forbiddenApisCli')
+
+        project.sourceSets.forEach { sourceSet ->
+            forbiddenApisCli.dependsOn(
+                project.tasks.create(sourceSet.getTaskName('forbiddenApisCli', null), JavaExec) {
+                    ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
+                    dependsOn(buildResources)
+                    classpath = project.files(
+                            project.configurations.forbiddenApisCliJar,
+                            sourceSet.compileClasspath,
+                            sourceSet.runtimeClasspath
+                    )
+                    main = 'de.thetaphi.forbiddenapis.cli.CliMain'
+                    executable = "${project.runtimeJavaHome}/bin/java"
+                    args "-b", 'jdk-unsafe-1.8'
+                    args "-b", 'jdk-deprecated-1.8'
+                    args "-b", 'jdk-non-portable'
+                    args "-b", 'jdk-system-out'
+                    args "-f", buildResources.copy("forbidden/jdk-signatures.txt")
+                    args "-f", buildResources.copy("forbidden/es-all-signatures.txt")
+                    args "--suppressannotation", '**.SuppressForbidden'
+                    if (sourceSet.name == 'test') {
+                        args "-f", buildResources.copy("forbidden/es-test-signatures.txt")
+                        args "-f", buildResources.copy("forbidden/http-signatures.txt")
+                    } else {
+                        args "-f", buildResources.copy("forbidden/es-server-signatures.txt")
+                    }
+                    dependsOn sourceSet.classesTaskName
+                    doFirst {
+                        // Forbidden APIs expects only existing dirs, and requires at least one
+                        FileCollection existingOutputs = sourceSet.output.classesDirs
+                                .filter { it.exists() }
+                        if (existingOutputs.isEmpty()) {
+                            throw new StopExecutionException("${sourceSet.name} has no outputs")
+                        }
+                        existingOutputs.forEach { args "-d", it }
+                    }
+                }
+            )
+        }
+        return forbiddenApisCli
+    }
+
     private static Task configureCheckstyle(Project project) {
         // Always copy the checkstyle configuration files to 'buildDir/checkstyle' since the resources could be located in a jar
         // file. If the resources are located in a jar, Gradle will fail when it tries to turn the URL into a file
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
index 390821c80ff39..a2484e9c5fce0 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
@@ -22,15 +22,14 @@ package org.elasticsearch.gradle.test
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin
 import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
-import org.gradle.api.Task
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.tasks.compile.JavaCompile
-
 /**
  * Configures the build to compile tests against Elasticsearch's test framework
  * and run REST tests. Use BuildPlugin if you want to build main code as well
@@ -48,6 +47,7 @@ public class StandaloneRestTestPlugin implements Plugin<Project> {
         project.pluginManager.apply(JavaBasePlugin)
         project.pluginManager.apply(RandomizedTestingPlugin)
 
+        project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
         BuildPlugin.globalBuildInfo(project)
         BuildPlugin.configureRepositories(project)
 
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTask.java
new file mode 100644
index 0000000000000..03c18f54e67ef
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTask.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.GradleException;
+import org.gradle.api.file.DirectoryProperty;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.SkipWhenEmpty;
+import org.gradle.api.tasks.StopExecutionException;
+import org.gradle.api.tasks.TaskAction;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Export Elasticsearch build resources to configurable paths
+ * <p>
+ * Wil overwrite existing files and create missing directories.
+ * Useful for resources that that need to be passed to other processes trough the filesystem or otherwise can't be
+ * consumed from the classpath.
+ */
+public class ExportElasticsearchBuildResourcesTask extends DefaultTask {
+
+    private final Logger logger = Logging.getLogger(ExportElasticsearchBuildResourcesTask.class);
+
+    private final Set<String> resources = new HashSet<>();
+
+    private DirectoryProperty outputDir;
+
+    public ExportElasticsearchBuildResourcesTask() {
+        outputDir = getProject().getLayout().directoryProperty(
+            getProject().getLayout().getBuildDirectory().dir("build-tools-exported")
+        );
+    }
+
+    @OutputDirectory
+    public DirectoryProperty getOutputDir() {
+        return outputDir;
+    }
+
+    @Input
+    @SkipWhenEmpty
+    public Set<String> getResources() {
+        return Collections.unmodifiableSet(resources);
+    }
+
+    @Classpath
+    public String getResourcesClasspath() {
+        // This will make sure the task is not considered up to date if the resources are changed.
+        logger.info("Classpath: {}", System.getProperty("java.class.path"));
+        return System.getProperty("java.class.path");
+    }
+
+    public void setOutputDir(DirectoryProperty outputDir) {
+        this.outputDir = outputDir;
+    }
+
+    public File copy(String resource) {
+        if (getState().getExecuted() || getState().getExecuting()) {
+            throw new GradleException("buildResources can't be configured after the task ran. " +
+                "Make sure task is not used after configuration time"
+            );
+        }
+        resources.add(resource);
+        return outputDir.file(resource).get().getAsFile();
+    }
+
+    @TaskAction
+    public void doExport() {
+        if (resources.isEmpty()) {
+            throw new StopExecutionException();
+        }
+        resources.stream().parallel()
+            .forEach(resourcePath -> {
+                Path destination = outputDir.get().file(resourcePath).getAsFile().toPath();
+                try (InputStream is = getClass().getClassLoader().getResourceAsStream(resourcePath)) {
+                    Files.createDirectories(destination.getParent());
+                    if (is == null) {
+                        throw new GradleException("Can't export `" + resourcePath + "` from build-tools: not found");
+                    }
+                    Files.copy(is, destination);
+                } catch (IOException e) {
+                    throw new GradleException("Can't write resource `" + resourcePath + "` to " + destination, e);
+                }
+            });
+    }
+
+}
diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 8ab68b40c0a6b..420ed3b10b4c9 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -686,6 +686,7 @@
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionScriptEngine.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]MoreExpressionTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]StoredExpressionTests.java" checks="LineLength" />
+  <suppress files="modules[/\\]lang-painless[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]painless[/\\]ContextExampleTests.java" checks="LineLength" />
   <suppress files="modules[/\\]reindex[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]reindex[/\\]TransportUpdateByQueryAction.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuCollationTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuFoldingTokenFilterFactory.java" checks="LineLength" />
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTaskIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTaskIT.java
new file mode 100644
index 0000000000000..98fea2ea15ab6
--- /dev/null
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTaskIT.java
@@ -0,0 +1,91 @@
+package org.elasticsearch.gradle;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+
+
+public class ExportElasticsearchBuildResourcesTaskIT extends GradleIntegrationTestCase {
+
+    public static final String PROJECT_NAME = "elasticsearch-build-resources";
+
+    public void testUpToDateWithSourcesConfigured() {
+        GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("clean", "-s")
+            .withPluginClasspath()
+            .build();
+
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("buildResources", "-s", "-i")
+            .withPluginClasspath()
+            .build();
+        assertTaskSuccessfull(result, ":buildResources");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
+
+        result = GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("buildResources", "-s", "-i")
+            .withPluginClasspath()
+            .build();
+        assertTaskUpToDate(result, ":buildResources");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
+    }
+
+    public void testImplicitTaskDependencyCopy() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("clean", "sampleCopyAll", "-s", "-i")
+            .withPluginClasspath()
+            .build();
+
+        assertTaskSuccessfull(result, ":buildResources");
+        assertTaskSuccessfull(result, ":sampleCopyAll");
+        assertBuildFileExists(result, PROJECT_NAME, "sampleCopyAll/checkstyle.xml");
+        // This is a side effect of compile time reference
+        assertBuildFileExists(result, PROJECT_NAME, "sampleCopyAll/checkstyle_suppressions.xml");
+    }
+
+    public void testImplicitTaskDependencyInputFileOfOther() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("clean", "sample", "-s", "-i")
+            .withPluginClasspath()
+            .build();
+
+        assertTaskSuccessfull(result, ":sample");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
+    }
+
+    public void testIncorrectUsage() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("noConfigAfterExecution", "-s", "-i")
+            .withPluginClasspath()
+            .buildAndFail();
+        assertOutputContains("buildResources can't be configured after the task ran");
+    }
+}
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
index 5c36fa61550d8..f00ab406a6c10 100644
--- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
@@ -1,8 +1,13 @@
 package org.elasticsearch.gradle.test;
 
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.BuildTask;
 import org.gradle.testkit.runner.GradleRunner;
+import org.gradle.testkit.runner.TaskOutcome;
 
 import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -15,7 +20,7 @@ protected File getProjectDir(String name) {
             throw new RuntimeException("Could not find resources dir for integration tests. " +
                 "Note that these tests can only be ran by Gradle and are not currently supported by the IDE");
         }
-        return new File(root, name);
+        return new File(root, name).getAbsoluteFile();
     }
 
     protected GradleRunner getGradleRunner(String sampleProject) {
@@ -61,4 +66,47 @@ protected void assertOutputDoesNotContain(String output, String... lines) {
         }
     }
 
+    protected void assertTaskSuccessfull(BuildResult result, String taskName) {
+        BuildTask task = result.task(taskName);
+        if (task == null) {
+            fail("Expected task `" + taskName + "` to be successful, but it did not run");
+        }
+        assertEquals(
+            "Expected task to be successful but it was: " + task.getOutcome() +
+                "\n\nOutput is:\n" + result.getOutput() ,
+            TaskOutcome.SUCCESS,
+            task.getOutcome()
+        );
+    }
+
+    protected void assertTaskUpToDate(BuildResult result, String taskName) {
+        BuildTask task = result.task(taskName);
+        if (task == null) {
+            fail("Expected task `" + taskName + "` to be up-to-date, but it did not run");
+        }
+        assertEquals(
+            "Expected task to be up to date but it was: " + task.getOutcome() +
+                "\n\nOutput is:\n" + result.getOutput() ,
+            TaskOutcome.UP_TO_DATE,
+            task.getOutcome()
+        );
+    }
+
+    protected void assertBuildFileExists(BuildResult result, String projectName, String path) {
+        Path absPath = getBuildDir(projectName).toPath().resolve(path);
+        assertTrue(
+            result.getOutput() + "\n\nExpected `" + absPath + "` to exists but it did not" +
+                "\n\nOutput is:\n" + result.getOutput(),
+            Files.exists(absPath)
+        );
+    }
+
+    protected void assertBuildFileDoesNotExists(BuildResult result, String projectName, String path) {
+        Path absPath = getBuildDir(projectName).toPath().resolve(path);
+        assertFalse(
+            result.getOutput() + "\n\nExpected `" + absPath + "` bo to exists but it did" +
+                "\n\nOutput is:\n" + result.getOutput(),
+            Files.exists(absPath)
+        );
+    }
 }
diff --git a/buildSrc/src/testKit/elasticsearch-build-resources/build.gradle b/buildSrc/src/testKit/elasticsearch-build-resources/build.gradle
new file mode 100644
index 0000000000000..95d1453025e92
--- /dev/null
+++ b/buildSrc/src/testKit/elasticsearch-build-resources/build.gradle
@@ -0,0 +1,38 @@
+plugins {
+    id 'elasticsearch.build'
+}
+
+ext.licenseFile = file("$buildDir/dummy/license")
+ext.noticeFile = file("$buildDir/dummy/notice")
+
+buildResources {
+    copy 'checkstyle.xml'
+}
+
+task sampleCopyAll(type: Sync) {
+    /** Note: no explicit dependency. This works with tasks that use the Provider API a.k.a "Lazy Configuration" **/
+    from buildResources
+    into "$buildDir/sampleCopyAll"
+}
+
+task sample {
+    // This does not work, task dependencies can't be providers
+    // dependsOn buildResources.resource('minimumRuntimeVersion')
+    // Nor does this, despite https://github.com/gradle/gradle/issues/3811
+    // dependsOn buildResources.outputDir
+    // for now it's just
+    dependsOn buildResources
+    // we have to refference it at configuration time in order to be picked up
+    ext.checkstyle_suppressions = buildResources.copy('checkstyle_suppressions.xml')
+    doLast {
+        println "This task is using ${file(checkstyle_suppressions)}"
+    }
+}
+
+task noConfigAfterExecution {
+    dependsOn buildResources
+    doLast {
+        println "This should cause an error because we are refferencing " +
+                "${buildResources.copy('checkstyle_suppressions.xml')} after the `buildResources` task has ran."
+    }
+}
\ No newline at end of file
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 023d5d5b8dc2b..34c266913d0a2 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,13 +1,15 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.5.0-snapshot-608f0277b0
+lucene            = 7.5.0-snapshot-13b9e28f9d
 
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.8.10
+jackson           = 2.8.11
 snakeyaml         = 1.17
+icu4j             = 62.1
+supercsv          = 2.4.0
 # when updating log4j, please update also docs/java-api/index.asciidoc
-log4j             = 2.9.1
+log4j             = 2.11.1
 slf4j             = 1.6.2
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle
diff --git a/client/benchmark/README.md b/client/benchmark/README.md
index 06211b9d8fe8c..68a910468e0cd 100644
--- a/client/benchmark/README.md
+++ b/client/benchmark/README.md
@@ -2,10 +2,18 @@
 
 1. Build `client-benchmark-noop-api-plugin` with `gradle :client:client-benchmark-noop-api-plugin:assemble`
 2. Install it on the target host with `bin/elasticsearch-plugin install file:///full/path/to/client-benchmark-noop-api-plugin.zip`
-3. Start Elasticsearch on the target host (ideally *not* on the same machine)
-4. Build an uberjar with `gradle :client:benchmark:shadowJar` and execute it.
+3. Start Elasticsearch on the target host (ideally *not* on the machine
+that runs the benchmarks)
+4. Run the benchmark with
+```
+./gradlew -p client/benchmark run --args ' params go here'
+```
 
-Repeat all steps above for the other benchmark candidate.
+Everything in the `'` gets sent on the command line to JMH. The leading ` `
+inside the `'`s is important. Without it parameters are sometimes sent to
+gradle.
+
+See below for some example invocations.
 
 ### Example benchmark
 
@@ -13,32 +21,35 @@ In general, you should define a few GC-related settings `-Xms8192M -Xmx8192M -XX
 
 #### Bulk indexing
 
-Download benchmark data from http://benchmarks.elastic.co/corpora/geonames/documents.json.bz2 and decompress them.
+Download benchmark data from http://benchmarks.elasticsearch.org.s3.amazonaws.com/corpora/geonames and decompress them.
 
-Example command line parameters:
+Example invocation:
 
 ```
-rest bulk 192.168.2.2 ./documents.json geonames type 8647880 5000
+wget http://benchmarks.elasticsearch.org.s3.amazonaws.com/corpora/geonames/documents-2.json.bz2
+bzip2 -d documents-2.json.bz2
+mv documents-2.json client/benchmark/build
+gradlew -p client/benchmark run --args ' rest bulk localhost build/documents-2.json geonames type 8647880 5000'
 ```
 
-The parameters are in order:
+The parameters are all in the `'`s and are in order:
 
 * Client type: Use either "rest" or "transport"
 * Benchmark type: Use either "bulk" or "search"
 * Benchmark target host IP (the host where Elasticsearch is running)
 * full path to the file that should be bulk indexed
 * name of the index
-* name of the (sole) type in the index 
+* name of the (sole) type in the index
 * number of documents in the file
 * bulk size
 
 
-#### Bulk indexing
+#### Search
 
-Example command line parameters:
+Example invocation:
 
 ```
-rest search 192.168.2.2 geonames "{ \"query\": { \"match_phrase\": { \"name\": \"Sankt Georgen\" } } }\"" 500,1000,1100,1200
+gradlew -p client/benchmark run --args ' rest search localhost geonames {"query":{"match_phrase":{"name":"Sankt Georgen"}}} 500,1000,1100,1200'
 ```
 
 The parameters are in order:
@@ -49,5 +60,3 @@ The parameters are in order:
 * name of the index
 * a search request body (remember to escape double quotes). The `TransportClientBenchmark` uses `QueryBuilders.wrapperQuery()` internally which automatically adds a root key `query`, so it must not be present in the command line parameter.
 * A comma-separated list of target throughput rates
-
-
diff --git a/client/benchmark/build.gradle b/client/benchmark/build.gradle
index 0c3238d985346..c67120c7cf59b 100644
--- a/client/benchmark/build.gradle
+++ b/client/benchmark/build.gradle
@@ -18,9 +18,6 @@
  */
 
 apply plugin: 'elasticsearch.build'
-// build an uberjar with all benchmarks
-apply plugin: 'com.github.johnrengelman.shadow'
-// have the shadow plugin provide the runShadow task
 apply plugin: 'application'
 
 group = 'org.elasticsearch.client'
@@ -32,7 +29,6 @@ build.dependsOn.remove('assemble')
 archivesBaseName = 'client-benchmarks'
 mainClassName = 'org.elasticsearch.client.benchmark.BenchmarkMain'
 
-
 // never try to invoke tests on the benchmark project - there aren't any
 test.enabled = false
 
diff --git a/client/benchmark/src/main/resources/log4j2.properties b/client/benchmark/src/main/resources/log4j2.properties
index 8652131bf4916..a68c7f5653f87 100644
--- a/client/benchmark/src/main/resources/log4j2.properties
+++ b/client/benchmark/src/main/resources/log4j2.properties
@@ -1,7 +1,7 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
index 65c5d094c7170..6f5eab6e1db1e 100644
--- a/client/rest-high-level/build.gradle
+++ b/client/rest-high-level/build.gradle
@@ -30,6 +30,14 @@ apply plugin: 'com.github.johnrengelman.shadow'
 group = 'org.elasticsearch.client'
 archivesBaseName = 'elasticsearch-rest-high-level-client'
 
+publishing {
+  publications {
+    nebula {
+      artifactId = archivesBaseName
+    }
+  }
+}
+
 //we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions)
 Task copyRestSpec = RestIntegTestTask.createCopyRestSpecTask(project, Providers.FALSE)
 test.dependsOn(copyRestSpec)
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java
index e889ec5beba80..99d50f6b46b7e 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 
 import java.io.IOException;
 
@@ -54,9 +54,9 @@ public final class IngestClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public WritePipelineResponse putPipeline(PutPipelineRequest request, RequestOptions options) throws IOException {
+    public AcknowledgedResponse putPipeline(PutPipelineRequest request, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::putPipeline, options,
-            WritePipelineResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -67,9 +67,9 @@ public WritePipelineResponse putPipeline(PutPipelineRequest request, RequestOpti
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
-    public void putPipelineAsync(PutPipelineRequest request, RequestOptions options, ActionListener<WritePipelineResponse> listener) {
+    public void putPipelineAsync(PutPipelineRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::putPipeline, options,
-            WritePipelineResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**
@@ -109,9 +109,9 @@ public void getPipelineAsync(GetPipelineRequest request, RequestOptions options,
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public WritePipelineResponse deletePipeline(DeletePipelineRequest request, RequestOptions options) throws IOException {
+    public AcknowledgedResponse deletePipeline(DeletePipelineRequest request, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::deletePipeline, options,
-            WritePipelineResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -123,9 +123,9 @@ public WritePipelineResponse deletePipeline(DeletePipelineRequest request, Reque
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
-    public void deletePipelineAsync(DeletePipelineRequest request, RequestOptions options, ActionListener<WritePipelineResponse> listener) {
+    public void deletePipelineAsync(DeletePipelineRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::deletePipeline, options,
-            WritePipelineResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseClient.java
index 587578f3b35e1..589f187f397aa 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseClient.java
@@ -19,11 +19,27 @@
 
 package org.elasticsearch.client;
 
+import org.apache.http.HttpEntity;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseResponse;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.GetLicenseResponse;
 import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 
 import static java.util.Collections.emptySet;
 
@@ -34,7 +50,7 @@
  * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html">
  * X-Pack Licensing APIs on elastic.co</a> for more information.
  */
-public class LicenseClient {
+public final class LicenseClient {
 
     private final RestHighLevelClient restHighLevelClient;
 
@@ -54,7 +70,7 @@ public PutLicenseResponse putLicense(PutLicenseRequest request, RequestOptions o
     }
 
     /**
-     * Asynchronously updates license for the cluster cluster.
+     * Asynchronously updates license for the cluster.
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
@@ -63,4 +79,79 @@ public void putLicenseAsync(PutLicenseRequest request, RequestOptions options, A
             PutLicenseResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Returns the current license for the cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetLicenseResponse getLicense(GetLicenseRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequest(request, RequestConverters::getLicense, options,
+            response -> new GetLicenseResponse(convertResponseToJson(response)), emptySet());
+    }
+
+    /**
+     * Asynchronously returns the current license for the cluster cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void getLicenseAsync(GetLicenseRequest request, RequestOptions options, ActionListener<GetLicenseResponse> listener) {
+        restHighLevelClient.performRequestAsync(request, RequestConverters::getLicense, options,
+            response -> new GetLicenseResponse(convertResponseToJson(response)), listener, emptySet());
+    }
+
+    /**
+     * Deletes license from the cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public DeleteLicenseResponse deleteLicense(DeleteLicenseRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::deleteLicense, options,
+            DeleteLicenseResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously deletes license from the cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void deleteLicenseAsync(DeleteLicenseRequest request, RequestOptions options, ActionListener<DeleteLicenseResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::deleteLicense, options,
+            DeleteLicenseResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Converts an entire response into a json string
+     *
+     * This is useful for responses that we don't parse on the client side, but instead work as string
+     * such as in case of the license JSON
+     */
+    static String convertResponseToJson(Response response) throws IOException {
+        HttpEntity entity = response.getEntity();
+        if (entity == null) {
+            throw new IllegalStateException("Response body expected but not returned");
+        }
+        if (entity.getContentType() == null) {
+            throw new IllegalStateException("Elasticsearch didn't return the [Content-Type] header, unable to parse response body");
+        }
+        XContentType xContentType = XContentType.fromMediaTypeOrFormat(entity.getContentType().getValue());
+        if (xContentType == null) {
+            throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType().getValue());
+        }
+        if (xContentType == XContentType.JSON) {
+            // No changes is required
+            return Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8));
+        } else {
+            // Need to convert into JSON
+            try (InputStream stream = response.getEntity().getContent();
+                 XContentParser parser = XContentFactory.xContent(xContentType).createParser(NamedXContentRegistry.EMPTY,
+                     DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) {
+                parser.nextToken();
+                XContentBuilder builder = XContentFactory.jsonBuilder();
+                builder.copyCurrentStructure(parser);
+                return Strings.toString(builder);
+            }
+        }
+    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
new file mode 100644
index 0000000000000..5244432a89407
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
+import org.elasticsearch.protocol.xpack.ml.PutJobResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+
+/**
+ * Machine Learning API client wrapper for the {@link RestHighLevelClient}
+ *
+ * <p>
+ * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-apis.html">
+ * X-Pack Machine Learning APIs </a> for additional information.
+ */
+public final class MachineLearningClient {
+
+    private final RestHighLevelClient restHighLevelClient;
+
+    MachineLearningClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    /**
+     * Creates a new Machine Learning Job
+     * <p>
+     * For additional info
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-job.html">ML PUT job documentation</a>
+     *
+     * @param request the PutJobRequest containing the {@link org.elasticsearch.protocol.xpack.ml.job.config.Job} settings
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return PutJobResponse with enclosed {@link org.elasticsearch.protocol.xpack.ml.job.config.Job} object
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public PutJobResponse putJob(PutJobRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            RequestConverters::putMachineLearningJob,
+            options,
+            PutJobResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Creates a new Machine Learning Job asynchronously and notifies listener on completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-job.html">ML PUT job documentation</a>
+     *
+     * @param request  the request containing the {@link org.elasticsearch.protocol.xpack.ml.job.config.Job} settings
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void putJobAsync(PutJobRequest request, RequestOptions options, ActionListener<PutJobResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            RequestConverters::putMachineLearningJob,
+            options,
+            PutJobResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MigrationClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MigrationClient.java
new file mode 100644
index 0000000000000..7da3832994768
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MigrationClient.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+
+/**
+ * A wrapper for the {@link RestHighLevelClient} that provides methods for
+ * accessing the Elastic License-related methods
+ * <p>
+ * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/migration-api.html">
+ * X-Pack Migration APIs on elastic.co</a> for more information.
+ */
+public final class MigrationClient {
+
+    private final RestHighLevelClient restHighLevelClient;
+
+    MigrationClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    /**
+     * Get Migration Assistance for one or more indices
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public IndexUpgradeInfoResponse getAssistance(IndexUpgradeInfoRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::getMigrationAssistance, options,
+            IndexUpgradeInfoResponse::fromXContent, Collections.emptySet());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index b9360877dfcad..45c70593fe826 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -39,12 +39,12 @@
 import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
@@ -78,8 +78,8 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.action.ingest.GetPipelineRequest;
-import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -96,7 +96,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContent;
@@ -107,9 +107,14 @@
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.rankeval.RankEvalRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
-import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
 import org.elasticsearch.script.mustache.SearchTemplateRequest;
@@ -421,8 +426,14 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                 BytesReference indexSource = indexRequest.source();
                 XContentType indexXContentType = indexRequest.getContentType();
 
-                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
-                    LoggingDeprecationHandler.INSTANCE, indexSource, indexXContentType)) {
+                try (XContentParser parser = XContentHelper.createParser(
+                        /*
+                         * EMPTY and THROW are fine here because we just call
+                         * copyCurrentStructure which doesn't touch the
+                         * registry or deprecation.
+                         */
+                        NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                        indexSource, indexXContentType)) {
                     try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                         builder.copyCurrentStructure(parser);
                         source = BytesReference.bytes(builder).toBytesRef();
@@ -1133,6 +1144,18 @@ static Request xPackWatcherPutWatch(PutWatchRequest putWatchRequest) {
         return request;
     }
 
+    static Request xPackWatcherDeleteWatch(DeleteWatchRequest deleteWatchRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("watcher")
+            .addPathPartAsIs("watch")
+            .addPathPart(deleteWatchRequest.getId())
+            .build();
+
+        Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
+        return request;
+    }
+
     static Request xpackUsage(XPackUsageRequest usageRequest) {
         Request request = new Request(HttpGet.METHOD_NAME, "/_xpack/usage");
         Params parameters = new Params(request);
@@ -1141,7 +1164,11 @@ static Request xpackUsage(XPackUsageRequest usageRequest) {
     }
 
     static Request putLicense(PutLicenseRequest putLicenseRequest) {
-        Request request = new Request(HttpPut.METHOD_NAME, "/_xpack/license");
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("license")
+            .build();
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
         Params parameters = new Params(request);
         parameters.withTimeout(putLicenseRequest.timeout());
         parameters.withMasterTimeout(putLicenseRequest.masterNodeTimeout());
@@ -1152,6 +1179,48 @@ static Request putLicense(PutLicenseRequest putLicenseRequest) {
         return request;
     }
 
+    static Request getLicense(GetLicenseRequest getLicenseRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("license")
+            .build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Params parameters = new Params(request);
+        parameters.withLocal(getLicenseRequest.local());
+        return request;
+    }
+
+    static Request deleteLicense(DeleteLicenseRequest deleteLicenseRequest) {
+        Request request = new Request(HttpDelete.METHOD_NAME, "/_xpack/license");
+        Params parameters = new Params(request);
+        parameters.withTimeout(deleteLicenseRequest.timeout());
+        parameters.withMasterTimeout(deleteLicenseRequest.masterNodeTimeout());
+        return request;
+    }
+
+    static Request putMachineLearningJob(PutJobRequest putJobRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("ml")
+            .addPathPartAsIs("anomaly_detectors")
+            .addPathPart(putJobRequest.getJob().getId())
+            .build();
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(putJobRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
+    static Request getMigrationAssistance(IndexUpgradeInfoRequest indexUpgradeInfoRequest) {
+        EndpointBuilder endpointBuilder = new EndpointBuilder()
+            .addPathPartAsIs("_xpack/migration/assistance")
+            .addCommaSeparatedPathParts(indexUpgradeInfoRequest.indices());
+        String endpoint = endpointBuilder.build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Params parameters = new Params(request);
+        parameters.withIndicesOptions(indexUpgradeInfoRequest.indicesOptions());
+        return request;
+    }
+
     private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
         return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
index c71bebf6903ca..50d7465ae334a 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -59,7 +59,7 @@
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ContextParser;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -163,8 +163,11 @@
 import org.elasticsearch.search.aggregations.pipeline.derivative.ParsedDerivative;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
 import org.elasticsearch.search.suggest.term.TermSuggestion;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -205,6 +208,10 @@ public class RestHighLevelClient implements Closeable {
     private final SnapshotClient snapshotClient = new SnapshotClient(this);
     private final TasksClient tasksClient = new TasksClient(this);
     private final XPackClient xPackClient = new XPackClient(this);
+    private final WatcherClient watcherClient = new WatcherClient(this);
+    private final LicenseClient licenseClient = new LicenseClient(this);
+    private final MigrationClient migrationClient = new MigrationClient(this);
+    private final MachineLearningClient machineLearningClient = new MachineLearningClient(this);
 
     /**
      * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the
@@ -296,18 +303,64 @@ public final TasksClient tasks() {
     }
 
     /**
-     * A wrapper for the {@link RestHighLevelClient} that provides methods for
-     * accessing the Elastic Licensed X-Pack APIs that are shipped with the
-     * default distribution of Elasticsearch. All of these APIs will 404 if run
-     * against the OSS distribution of Elasticsearch.
+     * Provides methods for accessing the Elastic Licensed X-Pack Info
+     * and Usage APIs that are shipped with the default distribution of
+     * Elasticsearch. All of these APIs will 404 if run against the OSS
+     * distribution of Elasticsearch.
      * <p>
-     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-api.html">
-     * X-Pack APIs on elastic.co</a> for more information.
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/info-api.html">
+     * Info APIs on elastic.co</a> for more information.
      */
     public final XPackClient xpack() {
         return xPackClient;
     }
 
+    /**
+     * Provides methods for accessing the Elastic Licensed Watcher APIs that
+     * are shipped with the default distribution of Elasticsearch. All of
+     * these APIs will 404 if run against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api.html">
+     * Watcher APIs on elastic.co</a> for more information.
+     */
+    public WatcherClient watcher() { return watcherClient; }
+
+    /**
+     * Provides methods for accessing the Elastic Licensed Licensing APIs that
+     * are shipped with the default distribution of Elasticsearch. All of
+     * these APIs will 404 if run against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html">
+     * Licensing APIs on elastic.co</a> for more information.
+     */
+    public LicenseClient license() { return licenseClient; }
+
+    /**
+     * Provides methods for accessing the Elastic Licensed Licensing APIs that
+     * are shipped with the default distribution of Elasticsearch. All of
+     * these APIs will 404 if run against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/migration-api.html">
+     * Migration APIs on elastic.co</a> for more information.
+     */
+    public MigrationClient migration() {
+        return migrationClient;
+    }
+
+    /**
+     * Provides methods for accessing the Elastic Licensed Machine Learning APIs that
+     * are shipped with the Elastic Stack distribution of Elasticsearch. All of
+     * these APIs will 404 if run against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-apis.html">
+     * Machine Learning APIs on elastic.co</a> for more information.
+     *
+     * @return the client wrapper for making Machine Learning API calls
+     */
+    public MachineLearningClient machineLearning() {
+        return machineLearningClient;
+    }
+
     /**
      * Executes a bulk request using the Bulk API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html">Bulk API on elastic.co</a>
@@ -1050,8 +1103,7 @@ protected final <Resp> Resp parseEntity(final HttpEntity entity,
         if (xContentType == null) {
             throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType().getValue());
         }
-        try (XContentParser parser = xContentType.xContent().createParser(registry,
-            LoggingDeprecationHandler.INSTANCE, entity.getContent())) {
+        try (XContentParser parser = xContentType.xContent().createParser(registry, DEPRECATION_HANDLER, entity.getContent())) {
             return entityParser.apply(parser);
         }
     }
@@ -1069,6 +1121,19 @@ static boolean convertExistsResponse(Response response) {
         return response.getStatusLine().getStatusCode() == 200;
     }
 
+    /**
+     * Ignores deprecation warnings. This is appropriate because it is only
+     * used to parse responses from Elasticsearch. Any deprecation warnings
+     * emitted there just mean that you are talking to an old version of
+     * Elasticsearch. There isn't anything you can do about the deprecation.
+     */
+    private static final DeprecationHandler DEPRECATION_HANDLER = new DeprecationHandler() {
+        @Override
+        public void usedDeprecatedName(String usedName, String modernName) {}
+        @Override
+        public void usedDeprecatedField(String usedName, String replacedWith) {}
+    };
+
     static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         Map<String, ContextParser<Object, ? extends Aggregation>> map = new HashMap<>();
         map.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c));
@@ -1119,11 +1184,11 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         List<NamedXContentRegistry.Entry> entries = map.entrySet().stream()
                 .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
                 .collect(Collectors.toList());
-        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(TermSuggestion.NAME),
+        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(TermSuggestionBuilder.SUGGESTION_NAME),
                 (parser, context) -> TermSuggestion.fromXContent(parser, (String)context)));
-        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(PhraseSuggestion.NAME),
+        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(PhraseSuggestionBuilder.SUGGESTION_NAME),
                 (parser, context) -> PhraseSuggestion.fromXContent(parser, (String)context)));
-        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(CompletionSuggestion.NAME),
+        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(CompletionSuggestionBuilder.SUGGESTION_NAME),
                 (parser, context) -> CompletionSuggestion.fromXContent(parser, (String)context)));
         return entries;
     }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java
index 73c92ba5c45d5..48487926f024b 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java
@@ -19,12 +19,15 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 
 import java.io.IOException;
 
 import static java.util.Collections.emptySet;
+import static java.util.Collections.singleton;
 
 public final class WatcherClient {
 
@@ -61,4 +64,31 @@ public void putWatchAsync(PutWatchRequest request, RequestOptions options,
         restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xPackWatcherPutWatch, options,
             PutWatchResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * Deletes a watch from the cluster
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-delete-watch.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public DeleteWatchResponse deleteWatch(DeleteWatchRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::xPackWatcherDeleteWatch, options,
+            DeleteWatchResponse::fromXContent, singleton(404));
+    }
+
+    /**
+     * Asynchronously deletes a watch from the cluster
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-delete-watch.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void deleteWatchAsync(DeleteWatchRequest request, RequestOptions options, ActionListener<DeleteWatchResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xPackWatcherDeleteWatch, options,
+            DeleteWatchResponse::fromXContent, listener, singleton(404));
+    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
index 1401376527df2..2af49ba1a1b73 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
@@ -41,17 +41,9 @@
 public final class XPackClient {
 
     private final RestHighLevelClient restHighLevelClient;
-    private final WatcherClient watcherClient;
-    private final LicenseClient licenseClient;
 
     XPackClient(RestHighLevelClient restHighLevelClient) {
         this.restHighLevelClient = restHighLevelClient;
-        this.watcherClient = new WatcherClient(restHighLevelClient);
-        this.licenseClient = new LicenseClient(restHighLevelClient);
-    }
-
-    public WatcherClient watcher() {
-        return watcherClient;
     }
 
     /**
@@ -102,15 +94,4 @@ public void usageAsync(XPackUsageRequest request, RequestOptions options, Action
         restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xpackUsage, options,
             XPackUsageResponse::fromXContent, listener, emptySet());
     }
-
-    /**
-     * A wrapper for the {@link RestHighLevelClient} that provides methods for
-     * accessing the Elastic Licensing APIs.
-     * <p>
-     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html">
-     * X-Pack APIs on elastic.co</a> for more information.
-     */
-    public LicenseClient license() {
-        return licenseClient;
-    }
 }
diff --git a/client/rest-high-level/src/main/resources/forbidden/rest-high-level-signatures.txt b/client/rest-high-level/src/main/resources/forbidden/rest-high-level-signatures.txt
index fb2330f3f083c..33e136a66f44f 100644
--- a/client/rest-high-level/src/main/resources/forbidden/rest-high-level-signatures.txt
+++ b/client/rest-high-level/src/main/resources/forbidden/rest-high-level-signatures.txt
@@ -19,3 +19,6 @@ org.apache.http.entity.ContentType#create(java.lang.String)
 org.apache.http.entity.ContentType#create(java.lang.String,java.lang.String)
 org.apache.http.entity.ContentType#create(java.lang.String,java.nio.charset.Charset)
 org.apache.http.entity.ContentType#create(java.lang.String,org.apache.http.NameValuePair[])
+
+@defaultMessage We can't rely on log4j2 being on the classpath so don't log deprecations!
+org.elasticsearch.common.xcontent.LoggingDeprecationHandler
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java
index 1f5914f392cf4..7068529619232 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java
@@ -28,7 +28,7 @@
 import org.elasticsearch.action.ingest.SimulateDocumentVerboseResult;
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -50,7 +50,7 @@ public void testPutPipeline() throws IOException {
             BytesReference.bytes(pipelineBuilder),
             pipelineBuilder.contentType());
 
-        WritePipelineResponse putPipelineResponse =
+        AcknowledgedResponse putPipelineResponse =
             execute(request, highLevelClient().ingest()::putPipeline, highLevelClient().ingest()::putPipelineAsync);
         assertTrue(putPipelineResponse.isAcknowledged());
     }
@@ -86,7 +86,7 @@ public void testDeletePipeline() throws IOException {
 
         DeletePipelineRequest request = new DeletePipelineRequest(id);
 
-        WritePipelineResponse response =
+        AcknowledgedResponse response =
             execute(request, highLevelClient().ingest()::deletePipeline, highLevelClient().ingest()::deletePipelineAsync);
         assertTrue(response.isAcknowledged());
     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
new file mode 100644
index 0000000000000..f86eb5b5dca87
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
+import org.elasticsearch.protocol.xpack.ml.PutJobResponse;
+import org.elasticsearch.protocol.xpack.ml.job.config.AnalysisConfig;
+import org.elasticsearch.protocol.xpack.ml.job.config.DataDescription;
+import org.elasticsearch.protocol.xpack.ml.job.config.Detector;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.is;
+
+public class MachineLearningIT extends ESRestHighLevelClientTestCase {
+
+    public void testPutJob() throws Exception {
+        String jobId = randomValidJobId();
+        Job job = buildJob(jobId);
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+
+        PutJobResponse putJobResponse = execute(new PutJobRequest(job), machineLearningClient::putJob, machineLearningClient::putJobAsync);
+        Job createdJob = putJobResponse.getResponse();
+
+        assertThat(createdJob.getId(), is(jobId));
+        assertThat(createdJob.getJobType(), is(Job.ANOMALY_DETECTOR_JOB_TYPE));
+    }
+
+    public static String randomValidJobId() {
+        CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz0123456789".toCharArray());
+        return generator.ofCodePointsLength(random(), 10, 10);
+    }
+
+    private static Job buildJob(String jobId) {
+        Job.Builder builder = new Job.Builder(jobId);
+        builder.setDescription(randomAlphaOfLength(10));
+
+        Detector detector = new Detector.Builder()
+            .setFieldName("total")
+            .setFunction("sum")
+            .setDetectorDescription(randomAlphaOfLength(10))
+            .build();
+        AnalysisConfig.Builder configBuilder = new AnalysisConfig.Builder(Arrays.asList(detector));
+        configBuilder.setBucketSpan(new TimeValue(randomIntBetween(1, 10), TimeUnit.SECONDS));
+        builder.setAnalysisConfig(configBuilder);
+
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setTimeFormat(randomFrom(DataDescription.EPOCH_MS, DataDescription.EPOCH));
+        dataDescription.setTimeField(randomAlphaOfLength(10));
+        builder.setDataDescription(dataDescription);
+
+        return builder.build();
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java
new file mode 100644
index 0000000000000..03614537bfe78
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoResponse;
+
+import java.io.IOException;
+
+public class MigrationIT extends ESRestHighLevelClientTestCase {
+
+    public void testGetAssistance() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            IndexUpgradeInfoResponse response = client.migration().getAssistance(new IndexUpgradeInfoRequest(), RequestOptions.DEFAULT);
+            assertEquals(0, response.getActions().size());
+        }
+        {
+            client.indices().create(new CreateIndexRequest("test"), RequestOptions.DEFAULT);
+            IndexUpgradeInfoResponse response = client.migration().getAssistance(
+                new IndexUpgradeInfoRequest("test"), RequestOptions.DEFAULT);
+            assertEquals(0, response.getActions().size());
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
index 0415d363c54cd..47195f0bb2aba 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -126,6 +126,8 @@
 import org.elasticsearch.index.rankeval.RatedRequest;
 import org.elasticsearch.index.rankeval.RestRankEvalAction;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.action.search.RestSearchAction;
@@ -2551,6 +2553,23 @@ public void testXPackInfo() {
         assertEquals(expectedParams, request.getParameters());
     }
 
+    public void testGetMigrationAssistance() {
+        IndexUpgradeInfoRequest upgradeInfoRequest = new IndexUpgradeInfoRequest();
+        String expectedEndpoint = "/_xpack/migration/assistance";
+        if (randomBoolean()) {
+            String[] indices = randomIndicesNames(1, 5);
+            upgradeInfoRequest.indices(indices);
+            expectedEndpoint += "/" + String.join(",", indices);
+        }
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomIndicesOptions(upgradeInfoRequest::indicesOptions, upgradeInfoRequest::indicesOptions, expectedParams);
+        Request request = RequestConverters.getMigrationAssistance(upgradeInfoRequest);
+        assertEquals(HttpGet.METHOD_NAME, request.getMethod());
+        assertEquals(expectedEndpoint, request.getEndpoint());
+        assertNull(request.getEntity());
+        assertEquals(expectedParams, request.getParameters());
+    }
+
     public void testXPackPutWatch() throws Exception {
         PutWatchRequest putWatchRequest = new PutWatchRequest();
         String watchId = randomAlphaOfLength(10);
@@ -2580,6 +2599,17 @@ public void testXPackPutWatch() throws Exception {
         assertThat(bos.toString("UTF-8"), is(body));
     }
 
+    public void testXPackDeleteWatch() {
+        DeleteWatchRequest deleteWatchRequest = new DeleteWatchRequest();
+        String watchId = randomAlphaOfLength(10);
+        deleteWatchRequest.setId(watchId);
+
+        Request request = RequestConverters.xPackWatcherDeleteWatch(deleteWatchRequest);
+        assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
+        assertEquals("/_xpack/watcher/watch/" + watchId, request.getEndpoint());
+        assertThat(request.getEntity(), nullValue());
+    }
+
     /**
      * Randomize the {@link FetchSourceContext} request parameters.
      */
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index 5cf3b35275620..b5d8dbb628eb9 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.client;
 
 import com.fasterxml.jackson.core.JsonParseException;
-
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
@@ -755,7 +754,11 @@ public void testApiNamingConventions() throws Exception {
                             method.isAnnotationPresent(Deprecated.class));
                     } else {
                         //TODO xpack api are currently ignored, we need to load xpack yaml spec too
-                        if (apiName.startsWith("xpack.") == false) {
+                        if (apiName.startsWith("xpack.") == false &&
+                            apiName.startsWith("license.") == false &&
+                            apiName.startsWith("machine_learning.") == false &&
+                            apiName.startsWith("watcher.") == false &&
+                            apiName.startsWith("migration.") == false) {
                             apiNotFound.add(apiName);
                         }
                     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
index 9c9c5425f0006..3151e9badb6de 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
@@ -59,6 +59,9 @@
 import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.significant.heuristics.PercentageScore;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.matrix.stats.MatrixStats;
@@ -267,6 +270,33 @@ public void testSearchWithTermsAgg() throws IOException {
         assertEquals(2, type2.getDocCount());
         assertEquals(0, type2.getAggregations().asList().size());
     }
+    
+    public void testSearchWithSignificantTermsAgg() throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.query(new MatchQueryBuilder("num","50"));
+        searchSourceBuilder.aggregation(new SignificantTermsAggregationBuilder("agg1", ValueType.STRING)
+                .field("type.keyword")
+                .minDocCount(1)
+                .significanceHeuristic(new PercentageScore()));
+        searchSourceBuilder.size(0);
+        searchRequest.source(searchSourceBuilder);
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+        assertSearchHeader(searchResponse);
+        assertNull(searchResponse.getSuggest());
+        assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
+        assertEquals(0, searchResponse.getHits().getHits().length);
+        assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
+        SignificantTerms significantTermsAgg = searchResponse.getAggregations().get("agg1");
+        assertEquals("agg1", significantTermsAgg.getName());
+        assertEquals(1, significantTermsAgg.getBuckets().size());
+        SignificantTerms.Bucket type1 = significantTermsAgg.getBucketByKey("type1");
+        assertEquals(1, type1.getDocCount());
+        assertEquals(1, type1.getSubsetDf());
+        assertEquals(1, type1.getSubsetSize());
+        assertEquals(3, type1.getSupersetDf());
+        assertEquals(1d/3d, type1.getSignificanceScore(), 0d);
+    }    
 
     public void testSearchWithRangeAgg() throws IOException {
         {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SyncedFlushResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SyncedFlushResponseTests.java
index 0756cfa6bab10..8057a92b3f279 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SyncedFlushResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SyncedFlushResponseTests.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -63,7 +63,7 @@ public void testXContentSerialization() throws IOException {
                 .xContent()
                 .createParser(
                     xContentRegistry(),
-                    LoggingDeprecationHandler.INSTANCE,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                     BytesReference.bytes(serverResponsebuilder).streamInput()
                 ).map()
         );
@@ -74,7 +74,7 @@ public void testXContentSerialization() throws IOException {
                 .xContent()
                 .createParser(
                     xContentRegistry(),
-                    LoggingDeprecationHandler.INSTANCE,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                     BytesReference.bytes(clientResponsebuilder).streamInput()
                 )
                 .map()
@@ -94,7 +94,9 @@ public void testXContentDeserialization() throws IOException {
             .contentType()
             .xContent()
             .createParser(
-                xContentRegistry(), LoggingDeprecationHandler.INSTANCE, BytesReference.bytes(builder).streamInput()
+                xContentRegistry(),
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                BytesReference.bytes(builder).streamInput()
             );
         SyncedFlushResponse originalResponse = plan.clientResult;
         SyncedFlushResponse parsedResponse = SyncedFlushResponse.fromXContent(parser);
@@ -175,7 +177,8 @@ TestPlan createTestPlan() throws IOException {
                                     .contentType()
                                     .xContent()
                                     .createParser(
-                                        xContentRegistry(), LoggingDeprecationHandler.INSTANCE,
+                                        xContentRegistry(),
+                                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                                         BytesReference.bytes(builder).streamInput()
                                     )
                                     .map();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java
index dec438a47ab18..491992735afbf 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java
@@ -21,6 +21,8 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 
@@ -30,6 +32,13 @@ public class WatcherIT extends ESRestHighLevelClientTestCase {
 
     public void testPutWatch() throws Exception {
         String watchId = randomAlphaOfLength(10);
+        PutWatchResponse putWatchResponse = createWatch(watchId);
+        assertThat(putWatchResponse.isCreated(), is(true));
+        assertThat(putWatchResponse.getId(), is(watchId));
+        assertThat(putWatchResponse.getVersion(), is(1L));
+    }
+
+    private PutWatchResponse createWatch(String watchId) throws Exception {
         String json = "{ \n" +
             "  \"trigger\": { \"schedule\": { \"interval\": \"10h\" } },\n" +
             "  \"input\": { \"none\": {} },\n" +
@@ -37,10 +46,30 @@ public void testPutWatch() throws Exception {
             "}";
         BytesReference bytesReference = new BytesArray(json);
         PutWatchRequest putWatchRequest = new PutWatchRequest(watchId, bytesReference, XContentType.JSON);
-        PutWatchResponse putWatchResponse = highLevelClient().xpack().watcher().putWatch(putWatchRequest, RequestOptions.DEFAULT);
-        assertThat(putWatchResponse.isCreated(), is(true));
-        assertThat(putWatchResponse.getId(), is(watchId));
-        assertThat(putWatchResponse.getVersion(), is(1L));
+        return highLevelClient().watcher().putWatch(putWatchRequest, RequestOptions.DEFAULT);
+    }
+
+    public void testDeleteWatch() throws Exception {
+        // delete watch that exists
+        {
+            String watchId = randomAlphaOfLength(10);
+            createWatch(watchId);
+            DeleteWatchResponse deleteWatchResponse = highLevelClient().watcher().deleteWatch(new DeleteWatchRequest(watchId),
+                RequestOptions.DEFAULT);
+            assertThat(deleteWatchResponse.getId(), is(watchId));
+            assertThat(deleteWatchResponse.getVersion(), is(2L));
+            assertThat(deleteWatchResponse.isFound(), is(true));
+        }
+
+        // delete watch that does not exist
+        {
+            String watchId = randomAlphaOfLength(10);
+            DeleteWatchResponse deleteWatchResponse = highLevelClient().watcher().deleteWatch(new DeleteWatchRequest(watchId),
+                RequestOptions.DEFAULT);
+            assertThat(deleteWatchResponse.getId(), is(watchId));
+            assertThat(deleteWatchResponse.getVersion(), is(1L));
+            assertThat(deleteWatchResponse.isFound(), is(false));
+        }
     }
 
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java
index 98502e3668af1..4702c34c6de3d 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
 import org.elasticsearch.action.ingest.SimulateProcessorResult;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
@@ -93,7 +93,7 @@ public void testPutPipeline() throws IOException {
             // end::put-pipeline-request-masterTimeout
 
             // tag::put-pipeline-execute
-            WritePipelineResponse response = client.ingest().putPipeline(request, RequestOptions.DEFAULT); // <1>
+            AcknowledgedResponse response = client.ingest().putPipeline(request, RequestOptions.DEFAULT); // <1>
             // end::put-pipeline-execute
 
             // tag::put-pipeline-response
@@ -117,10 +117,10 @@ public void testPutPipelineAsync() throws Exception {
             );
 
             // tag::put-pipeline-execute-listener
-            ActionListener<WritePipelineResponse> listener =
-                new ActionListener<WritePipelineResponse>() {
+            ActionListener<AcknowledgedResponse> listener =
+                new ActionListener<AcknowledgedResponse>() {
                     @Override
-                    public void onResponse(WritePipelineResponse response) {
+                    public void onResponse(AcknowledgedResponse response) {
                         // <1>
                     }
 
@@ -236,7 +236,7 @@ public void testDeletePipeline() throws IOException {
             // end::delete-pipeline-request-masterTimeout
 
             // tag::delete-pipeline-execute
-            WritePipelineResponse response = client.ingest().deletePipeline(request, RequestOptions.DEFAULT); // <1>
+            AcknowledgedResponse response = client.ingest().deletePipeline(request, RequestOptions.DEFAULT); // <1>
             // end::delete-pipeline-execute
 
             // tag::delete-pipeline-response
@@ -257,10 +257,10 @@ public void testDeletePipelineAsync() throws Exception {
             DeletePipelineRequest request = new DeletePipelineRequest("my-pipeline-id");
 
             // tag::delete-pipeline-execute-listener
-            ActionListener<WritePipelineResponse> listener =
-                new ActionListener<WritePipelineResponse>() {
+            ActionListener<AcknowledgedResponse> listener =
+                new ActionListener<AcknowledgedResponse>() {
                     @Override
-                    public void onResponse(WritePipelineResponse response) {
+                    public void onResponse(AcknowledgedResponse response) {
                         // <1>
                     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java
index d620adb71312b..6146030999c56 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java
@@ -19,11 +19,17 @@
 
 package org.elasticsearch.client.documentation;
 
+import org.elasticsearch.Build;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.Booleans;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseResponse;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.GetLicenseResponse;
 import org.elasticsearch.protocol.xpack.license.LicensesStatus;
 import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
@@ -32,6 +38,8 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.startsWith;
@@ -42,7 +50,8 @@
  */
 public class LicensingDocumentationIT extends ESRestHighLevelClientTestCase {
 
-    public void testPutLicense() throws Exception {
+    public void testLicense() throws Exception {
+        assumeTrue("License is only valid when tested against snapshot/test builds", Build.CURRENT.isSnapshot());
         RestHighLevelClient client = highLevelClient();
         String license = "{\"license\": {\"uid\":\"893361dc-9749-4997-93cb-802e3d7fa4a8\",\"type\":\"gold\"," +
             "\"issue_date_in_millis\":1411948800000,\"expiry_date_in_millis\":1914278399999,\"max_nodes\":1,\"issued_to\":\"issued_to\"," +
@@ -60,7 +69,7 @@ public void testPutLicense() throws Exception {
             request.setLicenseDefinition(license);  // <1>
             request.setAcknowledge(false);          // <2>
 
-            PutLicenseResponse response = client.xpack().license().putLicense(request, RequestOptions.DEFAULT);
+            PutLicenseResponse response = client.license().putLicense(request, RequestOptions.DEFAULT);
             //end::put-license-execute
 
             //tag::put-license-response
@@ -80,7 +89,7 @@ public void testPutLicense() throws Exception {
             // tag::put-license-execute-listener
             ActionListener<PutLicenseResponse> listener = new ActionListener<PutLicenseResponse>() {
                 @Override
-                public void onResponse(PutLicenseResponse indexResponse) {
+                public void onResponse(PutLicenseResponse putLicenseResponse) {
                     // <1>
                 }
 
@@ -96,11 +105,114 @@ public void onFailure(Exception e) {
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::put-license-execute-async
-            client.xpack().license().putLicenseAsync(
+            client.license().putLicenseAsync(
                     request, RequestOptions.DEFAULT, listener); // <1>
             // end::put-license-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
         }
+
+        // we cannot actually delete the license, otherwise the remaining tests won't work
+        if (Booleans.isTrue("true")) {
+            return;
+        }
+        {
+            //tag::delete-license-execute
+            DeleteLicenseRequest request = new DeleteLicenseRequest();
+
+            DeleteLicenseResponse response = client.license().deleteLicense(request, RequestOptions.DEFAULT);
+            //end::delete-license-execute
+
+            //tag::delete-license-response
+            boolean acknowledged = response.isAcknowledged(); // <1>
+            //end::delete-license-response
+
+            assertTrue(acknowledged);
+        }
+        {
+            DeleteLicenseRequest request = new DeleteLicenseRequest();
+            // tag::delete-license-execute-listener
+            ActionListener<DeleteLicenseResponse> listener = new ActionListener<DeleteLicenseResponse>() {
+                @Override
+                public void onResponse(DeleteLicenseResponse deleteLicenseResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::delete-license-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::delete-license-execute-async
+            client.license().deleteLicenseAsync(
+                request, RequestOptions.DEFAULT, listener); // <1>
+            // end::delete-license-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
+    public void testGetLicense() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        {
+            //tag::get-license-execute
+            GetLicenseRequest request = new GetLicenseRequest();
+
+            GetLicenseResponse response = client.license().getLicense(request, RequestOptions.DEFAULT);
+            //end::get-license-execute
+
+            //tag::get-license-response
+            String currentLicense = response.getLicenseDefinition(); // <1>
+            //end::get-license-response
+
+            assertThat(currentLicense, containsString("trial"));
+            assertThat(currentLicense, containsString("client_rest-high-level_integTestCluster"));
+        }
+        {
+            GetLicenseRequest request = new GetLicenseRequest();
+            // tag::get-license-execute-listener
+            ActionListener<GetLicenseResponse> listener = new ActionListener<GetLicenseResponse>() {
+                @Override
+                public void onResponse(GetLicenseResponse indexResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::get-license-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::get-license-execute-async
+            client.license().getLicenseAsync(
+                request, RequestOptions.DEFAULT, listener); // <1>
+            // end::get-license-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+        {
+            GetLicenseRequest request = new GetLicenseRequest();
+            RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
+            // Make sure that it still works in other formats
+            builder.addHeader("Accept", randomFrom("application/smile", "application/cbor"));
+            RequestOptions options = builder.build();
+            GetLicenseResponse response = client.license().getLicense(request, options);
+            String currentLicense = response.getLicenseDefinition();
+            assertThat(currentLicense, startsWith("{"));
+            assertThat(currentLicense, containsString("trial"));
+            assertThat(currentLicense, containsString("client_rest-high-level_integTestCluster"));
+            assertThat(currentLicense, endsWith("}"));
+        }
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationClientDocumentationIT.java
new file mode 100644
index 0000000000000..c8310be8053b2
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationClientDocumentationIT.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.documentation;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoResponse;
+import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * This class is used to generate the Java Migration API documentation.
+ * You need to wrap your code between two tags like:
+ * // tag::example
+ * // end::example
+ *
+ * Where example is your tag name.
+ *
+ * Then in the documentation, you can extract what is between tag and end tags with
+ * ["source","java",subs="attributes,callouts,macros"]
+ * --------------------------------------------------
+ * include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[example]
+ * --------------------------------------------------
+ *
+ * The column width of the code block is 84. If the code contains a line longer
+ * than 84, the line will be cut and a horizontal scroll bar will be displayed.
+ * (the code indentation of the tag is not included in the width)
+ */
+public class MigrationClientDocumentationIT extends ESRestHighLevelClientTestCase {
+
+    public void testGetAssistance() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+
+        // tag::get-assistance-request
+        IndexUpgradeInfoRequest request = new IndexUpgradeInfoRequest(); // <1>
+        // end::get-assistance-request
+
+        // tag::get-assistance-request-indices
+        request.indices("index1", "index2"); // <1>
+        // end::get-assistance-request-indices
+
+        request.indices(Strings.EMPTY_ARRAY);
+
+        // tag::get-assistance-request-indices-options
+        request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1>
+        // end::get-assistance-request-indices-options
+
+        // tag::get-assistance-execute
+        IndexUpgradeInfoResponse response = client.migration().getAssistance(request, RequestOptions.DEFAULT);
+        // end::get-assistance-execute
+
+        // tag::get-assistance-response
+        Map<String, UpgradeActionRequired> actions = response.getActions();
+        for (Map.Entry<String, UpgradeActionRequired> entry : actions.entrySet()) {
+            String index = entry.getKey(); // <1>
+            UpgradeActionRequired actionRequired = entry.getValue(); // <2>
+        }
+        // end::get-assistance-response
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java
index df51d896cda70..707997d1f3103 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java
@@ -26,6 +26,8 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 
@@ -34,7 +36,7 @@
 
 public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase {
 
-    public void testPutWatch() throws Exception {
+    public void testWatcher() throws Exception {
         RestHighLevelClient client = highLevelClient();
 
         {
@@ -47,7 +49,7 @@ public void testPutWatch() throws Exception {
                 "}");
             PutWatchRequest request = new PutWatchRequest("my_watch_id", watch, XContentType.JSON);
             request.setActive(false); // <1>
-            PutWatchResponse response = client.xpack().watcher().putWatch(request, RequestOptions.DEFAULT);
+            PutWatchResponse response = client.watcher().putWatch(request, RequestOptions.DEFAULT);
             //end::x-pack-put-watch-execute
 
             //tag::x-pack-put-watch-response
@@ -83,10 +85,51 @@ public void onFailure(Exception e) {
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::x-pack-put-watch-execute-async
-            client.xpack().watcher().putWatchAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.watcher().putWatchAsync(request, RequestOptions.DEFAULT, listener); // <1>
             // end::x-pack-put-watch-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
         }
+
+        {
+            //tag::x-pack-delete-watch-execute
+            DeleteWatchRequest request = new DeleteWatchRequest("my_watch_id");
+            DeleteWatchResponse response = client.watcher().deleteWatch(request, RequestOptions.DEFAULT);
+            //end::x-pack-delete-watch-execute
+
+            //tag::x-pack-delete-watch-response
+            String watchId = response.getId(); // <1>
+            boolean found = response.isFound(); // <2>
+            long version = response.getVersion(); // <3>
+            //end::x-pack-delete-watch-response
+        }
+
+        {
+            DeleteWatchRequest request = new DeleteWatchRequest("my_other_watch_id");
+            // tag::x-pack-delete-watch-execute-listener
+            ActionListener<DeleteWatchResponse> listener = new ActionListener<DeleteWatchResponse>() {
+                @Override
+                public void onResponse(DeleteWatchResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::x-pack-delete-watch-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::x-pack-delete-watch-execute-async
+            client.watcher().deleteWatchAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::x-pack-delete-watch-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
     }
+
 }
diff --git a/client/sniffer/licenses/jackson-core-2.8.10.jar.sha1 b/client/sniffer/licenses/jackson-core-2.8.10.jar.sha1
deleted file mode 100644
index a322d371e265e..0000000000000
--- a/client/sniffer/licenses/jackson-core-2.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-eb21a035c66ad307e66ec8fce37f5d50fd62d039
\ No newline at end of file
diff --git a/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1 b/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/distribution/packages/src/common/systemd/elasticsearch.service b/distribution/packages/src/common/systemd/elasticsearch.service
index 409f04f76d058..a4d67d8830a56 100644
--- a/distribution/packages/src/common/systemd/elasticsearch.service
+++ b/distribution/packages/src/common/systemd/elasticsearch.service
@@ -6,6 +6,7 @@ After=network-online.target
 
 [Service]
 RuntimeDirectory=elasticsearch
+PrivateTmp=true
 Environment=ES_HOME=/usr/share/elasticsearch
 Environment=ES_PATH_CONF=${path.conf}
 Environment=PID_DIR=/var/run/elasticsearch
diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties
index 589f529e83eaa..6de21cd48f67b 100644
--- a/distribution/src/config/log4j2.properties
+++ b/distribution/src/config/log4j2.properties
@@ -7,13 +7,13 @@ logger.action.level = debug
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
 
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
 appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.rolling.layout.type = PatternLayout
-appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %.-10000m%n
 appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
@@ -38,7 +38,7 @@ appender.deprecation_rolling.type = RollingFile
 appender.deprecation_rolling.name = deprecation_rolling
 appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_rolling.layout.type = PatternLayout
-appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %.-10000m%n
 appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
 appender.deprecation_rolling.policies.type = Policies
 appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
@@ -55,7 +55,7 @@ appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
 appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
 appender.index_search_slowlog_rolling.layout.type = PatternLayout
-appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %.-10000m%n
 appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%i.log.gz
 appender.index_search_slowlog_rolling.policies.type = Policies
 appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy
@@ -72,7 +72,7 @@ appender.index_indexing_slowlog_rolling.type = RollingFile
 appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
 appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
 appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
-appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %.-10000m%n
 appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%i.log.gz
 appender.index_indexing_slowlog_rolling.policies.type = Policies
 appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy
diff --git a/docs/build.gradle b/docs/build.gradle
index a67c0217490b3..029147bba2ff0 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -37,6 +37,10 @@ integTestCluster {
   extraConfigFile 'hunspell/en_US/en_US.dic', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic'
   // Whitelist reindexing from the local node so we can test it.
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
+
+  // TODO: remove this for 7.0, this exists to allow the doc examples in 6.x to continue using the defaults
+  systemProperty 'es.scripting.use_java_time', 'false'
+  systemProperty 'es.scripting.update.ctx_in_params', 'false'
 }
 
 // remove when https://github.com/elastic/elasticsearch/issues/31305 is fixed
diff --git a/docs/java-api/index.asciidoc b/docs/java-api/index.asciidoc
index 4fb7db4c4abf6..5c3a94d57f41c 100644
--- a/docs/java-api/index.asciidoc
+++ b/docs/java-api/index.asciidoc
@@ -81,7 +81,7 @@ You need to also include Log4j 2 dependencies:
 <dependency>
     <groupId>org.apache.logging.log4j</groupId>
     <artifactId>log4j-core</artifactId>
-    <version>2.9.1</version>
+    <version>2.11.1</version>
 </dependency>
 --------------------------------------------------
 
@@ -109,7 +109,7 @@ If you want to use another logger than Log4j 2, you can use http://www.slf4j.org
 <dependency>
     <groupId>org.apache.logging.log4j</groupId>
     <artifactId>log4j-to-slf4j</artifactId>
-    <version>2.9.1</version>
+    <version>2.11.1</version>
 </dependency>
 <dependency>
     <groupId>org.slf4j</groupId>
diff --git a/docs/java-api/query-dsl/percolate-query.asciidoc b/docs/java-api/query-dsl/percolate-query.asciidoc
index e1968ae456a5c..9afce0842b9ae 100644
--- a/docs/java-api/query-dsl/percolate-query.asciidoc
+++ b/docs/java-api/query-dsl/percolate-query.asciidoc
@@ -49,7 +49,7 @@ XContentBuilder docBuilder = XContentFactory.jsonBuilder().startObject();
 docBuilder.field("content", "This is amazing!");
 docBuilder.endObject(); //End of the JSON root object
 
-PercolateQueryBuilder percolateQuery = new PercolateQueryBuilder("query", "docs", docBuilder.bytes());
+PercolateQueryBuilder percolateQuery = new PercolateQueryBuilder("query", "docs", BytesReference.bytes(docBuilder));
 
 // Percolate, by executing the percolator query in the query dsl:
 SearchResponse response = client().prepareSearch("myIndexName")
diff --git a/docs/java-rest/high-level/licensing/delete-license.asciidoc b/docs/java-rest/high-level/licensing/delete-license.asciidoc
new file mode 100644
index 0000000000000..d9aec6e57a14a
--- /dev/null
+++ b/docs/java-rest/high-level/licensing/delete-license.asciidoc
@@ -0,0 +1,51 @@
+[[java-rest-high-delete-license]]
+=== Delete License
+
+[[java-rest-high-delete-license-execution]]
+==== Execution
+
+The license can be deleted using the `deleteLicense()` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[delete-license-execute]
+--------------------------------------------------
+
+[[java-rest-high-delete-license-response]]
+==== Response
+
+The returned `DeleteLicenseResponse` contains the `acknowledged` flag, which
+returns true if the request was processed by all nodes.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[delete-license-response]
+--------------------------------------------------
+<1> Check the acknowledge flag. It should be true if license deletion is acknowledged.
+
+[[java-rest-high-delete-license-async]]
+==== Asynchronous Execution
+
+This request can be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[delete-license-execute-async]
+--------------------------------------------------
+<1> The `DeleteLicenseRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `DeleteLicenseResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[delete-license-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
diff --git a/docs/java-rest/high-level/licensing/get-license.asciidoc b/docs/java-rest/high-level/licensing/get-license.asciidoc
new file mode 100644
index 0000000000000..17eb89450fb15
--- /dev/null
+++ b/docs/java-rest/high-level/licensing/get-license.asciidoc
@@ -0,0 +1,50 @@
+[[java-rest-high-get-license]]
+=== Get License
+
+[[java-rest-high-get-license-execution]]
+==== Execution
+
+The license can be added or updated using the `getLicense()` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[get-license-execute]
+--------------------------------------------------
+
+[[java-rest-high-get-license-response]]
+==== Response
+
+The returned `GetLicenseResponse` contains the license in the JSON format.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[get-license-response]
+--------------------------------------------------
+<1> The text of the license.
+
+[[java-rest-high-get-license-async]]
+==== Asynchronous Execution
+
+This request can be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[get-license-execute-async]
+--------------------------------------------------
+<1> The `GetLicenseRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `GetLicenseResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[get-license-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
diff --git a/docs/java-rest/high-level/migration/get-assistance.asciidoc b/docs/java-rest/high-level/migration/get-assistance.asciidoc
new file mode 100644
index 0000000000000..20f857eb1fb41
--- /dev/null
+++ b/docs/java-rest/high-level/migration/get-assistance.asciidoc
@@ -0,0 +1,49 @@
+[[java-rest-high-migration-get-assistance]]
+=== Migration Get Assistance
+
+[[java-rest-high-migraton-get-assistance-request]]
+==== Index Upgrade Info Request
+
+An `IndexUpgradeInfoRequest` does not require any argument:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[get-assistance-request]
+--------------------------------------------------
+<1> Create a new request instance
+
+==== Optional arguments
+The following arguments can optionally be provided:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[get-assistance-request-indices]
+--------------------------------------------------
+<1> Set the indices to the request
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[get-assistance-request-indices-options]
+--------------------------------------------------
+<1> Set the `IndicesOptions` to control how unavailable indices are resolved and
+how wildcard expressions are expanded
+
+[[java-rest-high-migration-get-assistance-execution]]
+==== Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[get-assistance-execute]
+--------------------------------------------------
+
+[[java-rest-high-migration-get-assistance-response]]
+==== Response
+
+The returned `IndexUpgradeInfoResponse` contains the actions required for each index.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[get-assistance-response]
+--------------------------------------------------
+<1> Retrieve the index
+<2> Retrieve the action required for the migration of the current index
diff --git a/docs/java-rest/high-level/x-pack/x-pack-info.asciidoc b/docs/java-rest/high-level/miscellaneous/x-pack-info.asciidoc
similarity index 100%
rename from docs/java-rest/high-level/x-pack/x-pack-info.asciidoc
rename to docs/java-rest/high-level/miscellaneous/x-pack-info.asciidoc
diff --git a/docs/java-rest/high-level/x-pack/x-pack-usage.asciidoc b/docs/java-rest/high-level/miscellaneous/x-pack-usage.asciidoc
similarity index 98%
rename from docs/java-rest/high-level/x-pack/x-pack-usage.asciidoc
rename to docs/java-rest/high-level/miscellaneous/x-pack-usage.asciidoc
index 0927ae71c0bf5..c1e5ccf13e222 100644
--- a/docs/java-rest/high-level/x-pack/x-pack-usage.asciidoc
+++ b/docs/java-rest/high-level/miscellaneous/x-pack-usage.asciidoc
@@ -12,7 +12,7 @@ retrieved using the `usage()` method:
 include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-usage-execute]
 --------------------------------------------------
 
-[[java-rest-high-x-pack-info-response]]
+[[java-rest-high-x-pack-usage-response]]
 ==== Response
 
 The returned `XPackUsageResponse` contains a `Map` keyed by feature name.
diff --git a/docs/java-rest/high-level/search/explain.asciidoc b/docs/java-rest/high-level/search/explain.asciidoc
index 9e55ad77ea203..4d8c04448b1ae 100644
--- a/docs/java-rest/high-level/search/explain.asciidoc
+++ b/docs/java-rest/high-level/search/explain.asciidoc
@@ -80,7 +80,7 @@ A typical listener for `ExplainResponse` is constructed as follows:
 include-tagged::{doc-tests}/SearchDocumentationIT.java[explain-execute-listener]
 --------------------------------------------------
 <1> Called when the execution is successfully completed.
-<2> Called when the whole `FieldCapabilitiesRequest` fails.
+<2> Called when the whole `ExplainRequest` fails.
 
 [[java-rest-high-explain-response]]
 ==== ExplainResponse
diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc
index 25fbcaaaeaa73..9d7d66434f7e7 100644
--- a/docs/java-rest/high-level/supported-apis.asciidoc
+++ b/docs/java-rest/high-level/supported-apis.asciidoc
@@ -54,11 +54,12 @@ The Java High Level REST Client supports the following Miscellaneous APIs:
 * <<java-rest-high-main>>
 * <<java-rest-high-ping>>
 * <<java-rest-high-x-pack-info>>
+* <<java-rest-high-x-pack-usage>>
 
 include::miscellaneous/main.asciidoc[]
 include::miscellaneous/ping.asciidoc[]
-include::x-pack/x-pack-info.asciidoc[]
-include::x-pack/watcher/put-watch.asciidoc[]
+include::miscellaneous/x-pack-info.asciidoc[]
+include::miscellaneous/x-pack-usage.asciidoc[]
 
 == Indices APIs
 
@@ -187,11 +188,32 @@ The Java High Level REST Client supports the following Scripts APIs:
 include::script/get_script.asciidoc[]
 include::script/delete_script.asciidoc[]
 
-
 == Licensing APIs
 
 The Java High Level REST Client supports the following Licensing APIs:
 
 * <<java-rest-high-put-license>>
+* <<java-rest-high-get-license>>
+* <<java-rest-high-delete-license>>
 
 include::licensing/put-license.asciidoc[]
+include::licensing/get-license.asciidoc[]
+include::licensing/delete-license.asciidoc[]
+
+== Migration APIs
+
+The Java High Level REST Client supports the following Migration APIs:
+
+* <<java-rest-high-migration-get-assistance>>
+
+include::migration/get-assistance.asciidoc[]
+
+== Watcher APIs
+
+The Java High Level REST Client supports the following Watcher APIs:
+
+* <<java-rest-high-x-pack-watcher-put-watch>>
+* <<java-rest-high-x-pack-watcher-delete-watch>>
+
+include::watcher/put-watch.asciidoc[]
+include::watcher/delete-watch.asciidoc[]
diff --git a/docs/java-rest/high-level/watcher/delete-watch.asciidoc b/docs/java-rest/high-level/watcher/delete-watch.asciidoc
new file mode 100644
index 0000000000000..615337ba317bf
--- /dev/null
+++ b/docs/java-rest/high-level/watcher/delete-watch.asciidoc
@@ -0,0 +1,53 @@
+[[java-rest-high-x-pack-watcher-delete-watch]]
+=== Delete Watch API
+
+[[java-rest-high-x-pack-watcher-delete-watch-execution]]
+==== Execution
+
+A watch can be deleted as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-delete-watch-execute]
+--------------------------------------------------
+
+[[java-rest-high-x-pack-watcher-delete-watch-response]]
+==== Response
+
+The returned `DeleteWatchResponse` contains `found`, `id`,
+and `version` information.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-put-watch-response]
+--------------------------------------------------
+<1> `_id` contains id of the watch
+<2> `found` is a boolean indicating whether the watch was found
+<3> `_version` returns the version of the deleted watch
+
+[[java-rest-high-x-pack-watcher-delete-watch-async]]
+==== Asynchronous Execution
+
+This request can be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-delete-watch-execute-async]
+--------------------------------------------------
+<1> The `DeleteWatchRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `DeleteWatchResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-delete-watch-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
diff --git a/docs/java-rest/high-level/x-pack/watcher/put-watch.asciidoc b/docs/java-rest/high-level/watcher/put-watch.asciidoc
similarity index 90%
rename from docs/java-rest/high-level/x-pack/watcher/put-watch.asciidoc
rename to docs/java-rest/high-level/watcher/put-watch.asciidoc
index c803c54eb5e92..e5ee87bea34a6 100644
--- a/docs/java-rest/high-level/x-pack/watcher/put-watch.asciidoc
+++ b/docs/java-rest/high-level/watcher/put-watch.asciidoc
@@ -1,5 +1,5 @@
 [[java-rest-high-x-pack-watcher-put-watch]]
-=== X-Pack Info API
+=== Put Watch API
 
 [[java-rest-high-x-pack-watcher-put-watch-execution]]
 ==== Execution
@@ -16,7 +16,7 @@ include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-put-watch-execute
 [[java-rest-high-x-pack-watcher-put-watch-response]]
 ==== Response
 
-The returned `XPackPutWatchResponse` contain `created`, `id`,
+The returned `PutWatchResponse` contains `created`, `id`,
 and `version` information.
 
 ["source","java",subs="attributes,callouts,macros"]
@@ -36,7 +36,7 @@ This request can be executed asynchronously:
 --------------------------------------------------
 include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-put-watch-execute-async]
 --------------------------------------------------
-<1> The `XPackPutWatchRequest` to execute and the `ActionListener` to use when
+<1> The `PutWatchRequest` to execute and the `ActionListener` to use when
 the execution completes
 
 The asynchronous method does not block and returns immediately. Once it is
@@ -44,7 +44,7 @@ completed the `ActionListener` is called back using the `onResponse` method
 if the execution successfully completed or using the `onFailure` method if
 it failed.
 
-A typical listener for `XPackPutWatchResponse` looks like:
+A typical listener for `PutWatchResponse` looks like:
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
diff --git a/docs/painless/painless-contexts.asciidoc b/docs/painless/painless-contexts.asciidoc
index 8b8a3b0eec6b4..cc7bc752ec6d9 100644
--- a/docs/painless/painless-contexts.asciidoc
+++ b/docs/painless/painless-contexts.asciidoc
@@ -14,6 +14,8 @@ specialized code may define new ways to use a Painless script.
 |====
 | Name                              | Painless Documentation
                                     | Elasticsearch Documentation
+| Ingest processor                  | <<painless-ingest-processor-context, Painless Documentation>>
+                                    | {ref}/script-processor.html[Elasticsearch Documentation]
 | Update                            | <<painless-update-context, Painless Documentation>>
                                     | {ref}/docs-update.html[Elasticsearch Documentation]
 | Update by query                   | <<painless-update-by-query-context, Painless Documentation>>
@@ -44,12 +46,12 @@ specialized code may define new ways to use a Painless script.
                                     | {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
 | Bucket aggregation                | <<painless-bucket-agg-context, Painless Documentation>>
                                     | {ref}/search-aggregations-pipeline-bucket-script-aggregation.html[Elasticsearch Documentation]
-| Ingest processor                  | <<painless-ingest-processor-context, Painless Documentation>>
-                                    | {ref}/script-processor.html[Elasticsearch Documentation]
 | Watcher condition                 | <<painless-watcher-condition-context, Painless Documentation>>
                                     | {xpack-ref}/condition-script.html[Elasticsearch Documentation]
 | Watcher transform                 | <<painless-watcher-transform-context, Painless Documentation>>
                                     | {xpack-ref}/transform-script.html[Elasticsearch Documentation]
 |====
 
+include::painless-contexts/painless-context-examples.asciidoc[]
+
 include::painless-contexts/index.asciidoc[]
diff --git a/docs/painless/painless-contexts/index.asciidoc b/docs/painless/painless-contexts/index.asciidoc
index 64e4326e052f2..a9d3982133e1b 100644
--- a/docs/painless/painless-contexts/index.asciidoc
+++ b/docs/painless/painless-contexts/index.asciidoc
@@ -1,3 +1,5 @@
+include::painless-ingest-processor-context.asciidoc[]
+
 include::painless-update-context.asciidoc[]
 
 include::painless-update-by-query-context.asciidoc[]
@@ -28,8 +30,6 @@ include::painless-metric-agg-reduce-context.asciidoc[]
 
 include::painless-bucket-agg-context.asciidoc[]
 
-include::painless-ingest-processor-context.asciidoc[]
-
 include::painless-watcher-condition-context.asciidoc[]
 
 include::painless-watcher-transform-context.asciidoc[]
diff --git a/docs/painless/painless-contexts/painless-context-examples.asciidoc b/docs/painless/painless-contexts/painless-context-examples.asciidoc
new file mode 100644
index 0000000000000..469f425d1d89f
--- /dev/null
+++ b/docs/painless/painless-contexts/painless-context-examples.asciidoc
@@ -0,0 +1,80 @@
+[[painless-context-examples]]
+=== Context examples
+
+To run the examples, index the sample seat data into Elasticsearch. The examples
+must be run sequentially to work correctly.
+
+. Download the
+https://download.elastic.co/demos/painless/contexts/seats.json[seat data]. This
+data set contains booking information for a collection of plays. Each document
+represents a single seat for a play at a particular theater on a specific date
+and time.
++
+Each document contains the following fields:
++
+`theatre` ({ref}/keyword.html[`keyword`])::
+        The name of the theater the play is in.
+`play` ({ref}/text.html[`text`])::
+        The name of the play.
+`actors` ({ref}/text.html[`text`])::
+        A list of actors in the play.
+`row` ({ref}/number.html[`integer`])::
+        The row of the seat.
+`number` ({ref}/number.html[`integer`])::
+        The number of the seat within a row.
+`cost` ({ref}/number.html[`double`])::
+        The cost of the ticket for the seat.
+`sold` ({ref}/boolean.html[`boolean`])::
+        Whether or not the seat is sold.
+`datetime` ({ref}/date.html[`date`])::
+        The date and time of the play as a date object.
+`date` ({ref}/keyword.html[`keyword`])::
+        The date of the play as a keyword.
+`time` ({ref}/keyword.html[`keyword`])::
+        The time of the play as a keyword.
+
+. {defguide}/running-elasticsearch.html[Start] Elasticsearch. Note these
+examples assume Elasticsearch and Kibana are running locally. To use the Console
+editor with a remote Kibana instance, click the settings icon and enter the
+Console URL. To submit a cURL request to a remote Elasticsearch instance, edit
+the request URL.
+
+. Create {ref}/mapping.html[mappings] for the sample data:
++
+[source,js]
+----
+PUT /seats
+{
+  "mappings": {
+    "seat": {
+      "properties": {
+        "theatre":  { "type": "keyword" },
+        "play":     { "type": "text"    },
+        "actors":   { "type": "text"    },
+        "row":      { "type": "integer" },
+        "number":   { "type": "integer" },
+        "cost":     { "type": "double"  },
+        "sold":     { "type": "boolean" },
+        "datetime": { "type": "date"    },
+        "date":     { "type": "keyword" },
+        "time":     { "type": "keyword" }
+      }
+    }
+  }
+}
+----
++
+// CONSOLE
+
+. Run the <<painless-ingest-processor-context, ingest processor context>>
+example. This sets up a script ingest processor used on each document as the
+seat data is indexed.
+
+. Index the seat data:
++
+[source,js]
+----
+curl -XPOST localhost:9200/seats/seat/_bulk?pipeline=seats -H "Content-Type: application/x-ndjson" --data-binary "@/<local-file-path>/seats.json"
+----
+// NOTCONSOLE
+
diff --git a/docs/painless/painless-contexts/painless-ingest-processor-context.asciidoc b/docs/painless/painless-contexts/painless-ingest-processor-context.asciidoc
index 5d451268dedcd..546057ab1a0b8 100644
--- a/docs/painless/painless-contexts/painless-ingest-processor-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-ingest-processor-context.asciidoc
@@ -27,7 +27,7 @@ to modify documents upon insertion.
 {ref}/mapping-type-field.html[`ctx['_type']`]::
         Modify this to change the type for the current document.
 
-`ctx` (`Map`, read-only)::
+`ctx` (`Map`)::
         Modify the values in the `Map/List` structure to add, modify, or delete
         the fields of a document.
 
@@ -38,4 +38,158 @@ void::
 
 *API*
 
-The standard <<painless-api-reference, Painless API>> is available.
\ No newline at end of file
+The standard <<painless-api-reference, Painless API>> is available.
+
+*Example*
+
+To run this example, first follow the steps in
+<<painless-context-examples, context examples>>.
+
+The seat data contains:
+
+* A date in the format `YYYY-MM-DD` where the second digit of both month and day
+  is optional.
+* A time in the format HH:MM* where the second digit of both hours and minutes
+  is optional. The star (*) represents either the `String` `AM` or `PM`.
+
+The following ingest script processes the date and time `Strings` and stores the
+result in a `datetime` field.
+
+[source,Painless]
+----
+String[] split(String s, char d) {                                   <1>
+    int count = 0;
+
+    for (char c : s.toCharArray()) {                                 <2>
+        if (c == d) {
+            ++count;
+        }
+    }
+
+    if (count == 0) {
+        return new String[] {s};                                     <3>
+    }
+
+    String[] r = new String[count + 1];                              <4>
+    int i0 = 0, i1 = 0;
+    count = 0;
+
+    for (char c : s.toCharArray()) {                                 <5>
+        if (c == d) {
+            r[count++] = s.substring(i0, i1);
+            i0 = i1 + 1;
+        }
+
+        ++i1;
+    }
+
+    r[count] = s.substring(i0, i1);                                  <6>
+
+    return r;
+}
+
+String[] dateSplit = split(ctx.date, (char)"-");                     <7>
+String year = dateSplit[0].trim();
+String month = dateSplit[1].trim();
+
+if (month.length() == 1) {                                           <8>
+    month = "0" + month;
+}
+
+String day = dateSplit[2].trim();
+
+if (day.length() == 1) {                                             <9>
+    day = "0" + day;
+}
+
+boolean pm = ctx.time.substring(ctx.time.length() - 2).equals("PM"); <10>
+String[] timeSplit = split(
+        ctx.time.substring(0, ctx.time.length() - 2), (char)":");    <11>
+int hours = Integer.parseInt(timeSplit[0].trim());
+int minutes = Integer.parseInt(timeSplit[1].trim());
+
+if (pm) {                                                            <12>
+    hours += 12;
+}
+
+String dts = year + "-" + month + "-" + day + "T" +
+        (hours < 10 ? "0" + hours : "" + hours) + ":" +
+        (minutes < 10 ? "0" + minutes : "" + minutes) +
+        ":00+08:00";                                                 <13>
+
+ZonedDateTime dt = ZonedDateTime.parse(
+         dts, DateTimeFormatter.ISO_OFFSET_DATE_TIME);               <14>
+ctx.datetime = dt.getLong(ChronoField.INSTANT_SECONDS)*1000L;        <15>
+----
+<1> Creates a `split` <<painless-functions, function>> to split a
+    <<string-type, `String`>> type value using a <<primitive-types, `char`>>
+    type value as the delimiter. This is useful for handling the necessity of
+    pulling out the individual pieces of the date and time `Strings` from the
+    original seat data.
+<2> The first pass through each `char` in the `String` collects how many new
+    `Strings` the original is split into.
+<3> Returns the original `String` if there are no instances of the delimiting
+    `char`.
+<4> Creates an <<array-type, array type>> value to collect the split `Strings`
+    into based on the number of `char` delimiters found in the first pass.
+<5> The second pass through each `char` in the `String` collects each split
+    substring into an array type value of `Strings`.
+<6> Collects the last substring into the array type value of `Strings`.
+<7> Uses the `split` function to separate the date `String` from the seat data
+    into year, month, and day `Strings`.
+    Note::
+    * The use of a `String` type value to `char` type value
+      <<string-character-casting, cast>> as part of the second argument since
+      character literals do not exist.
+    * The use of the `ctx` ingest processor context variable to retrieve the
+      data from the `date` field.
+<8> Appends the <<string-literals, string literal>> `"0"` value to a single
+    digit month since the format of the seat data allows for this case.
+<9> Appends the <<string-literals, string literal>> `"0"` value to a single
+    digit day since the format of the seat data allows for this case.
+<10> Sets the <<primitive-types, `boolean type`>>
+     <<painless-variables, variable>> to `true` if the time `String` is a time
+     in the afternoon or evening.
+     Note::
+     * The use of the `ctx` ingest processor context variable to retrieve the
+       data from the `time` field.
+<11> Uses the `split` function to separate the time `String` from the seat data
+     into hours and minutes `Strings`.
+     Note::
+     * The use of the `substring` method to remove the `AM` or `PM` portion of
+       the time `String`.
+     * The use of a `String` type value to `char` type value
+       <<string-character-casting, cast>> as part of the second argument since
+       character literals do not exist.
+     * The use of the `ctx` ingest processor context variable to retrieve the
+       data from the `date` field.
+<12> If the time `String` is an afternoon or evening value adds the
+     <<integer-literals, integer literal>> `12` to the existing hours to move to
+     a 24-hour based time.
+<13> Builds a new time `String` that is parsable using existing API methods.
+<14> Creates a `ZonedDateTime` <<reference-types, reference type>> value by using
+     the API method `parse` to parse the new time `String`.
+<15> Sets the datetime field `datetime` to the number of milliseconds retrieved
+     from the API method `getLong`.
+     Note::
+     * The use of the `ctx` ingest processor context variable to set the field
+       `datetime`. Manipulate each document's fields with the `ctx` variable as
+       each document is indexed.
+
+Submit the following request:
+
+[source,js]
+----
+PUT /_ingest/pipeline/seats
+{
+    "description": "update datetime for seats",
+    "processors": [
+      {
+        "script": {
+          "source": "String[] split(String s, char d) { int count = 0; for (char c : s.toCharArray()) { if (c == d) { ++count; } } if (count == 0) { return new String[] {s}; } String[] r = new String[count + 1]; int i0 = 0, i1 = 0; count = 0; for (char c : s.toCharArray()) { if (c == d) { r[count++] = s.substring(i0, i1); i0 = i1 + 1; } ++i1; } r[count] = s.substring(i0, i1); return r; } String[] dateSplit = split(ctx.date, (char)\"-\"); String year = dateSplit[0].trim(); String month = dateSplit[1].trim(); if (month.length() == 1) { month = \"0\" + month; } String day = dateSplit[2].trim(); if (day.length() == 1) { day = \"0\" + day; } boolean pm = ctx.time.substring(ctx.time.length() - 2).equals(\"PM\"); String[] timeSplit = split(ctx.time.substring(0, ctx.time.length() - 2), (char)\":\"); int hours = Integer.parseInt(timeSplit[0].trim()); int minutes = Integer.parseInt(timeSplit[1].trim()); if (pm) { hours += 12; } String dts = year + \"-\" + month + \"-\" + day + \"T\" + (hours < 10 ? \"0\" + hours : \"\" + hours) + \":\" + (minutes < 10 ? \"0\" + minutes : \"\" + minutes) + \":00+08:00\"; ZonedDateTime dt = ZonedDateTime.parse(dts, DateTimeFormatter.ISO_OFFSET_DATE_TIME); ctx.datetime = dt.getLong(ChronoField.INSTANT_SECONDS)*1000L;"
+        }
+      }
+    ]
+}
+----
+// CONSOLE
\ No newline at end of file
diff --git a/docs/painless/painless-contexts/painless-metric-agg-combine-context.asciidoc b/docs/painless/painless-contexts/painless-metric-agg-combine-context.asciidoc
index 31cb596ae8167..5cc9ad8ecbb93 100644
--- a/docs/painless/painless-contexts/painless-metric-agg-combine-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-metric-agg-combine-context.asciidoc
@@ -12,7 +12,7 @@ optional as part of a full metric aggregation.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-`params['_agg']` (`Map`)::
+`state` (`Map`)::
         `Map` with values available from the prior map script.
 
 *Return*
diff --git a/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc b/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc
index 1503e3abb5891..8c0fddfa33961 100644
--- a/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc
@@ -12,13 +12,13 @@ full metric aggregation.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-`params['_agg']` (`Map`)::
+`state` (`Map`)::
         Empty `Map` used to add values for use in a
         <<painless-metric-agg-map-context, map script>>.
 
 *Side Effects*
 
-`params['_agg']` (`Map`)::
+`state` (`Map`)::
         Add values to this `Map` to for use in a map.  Additional values must
         be of the type `Map`, `List`, `String` or primitive.
 
diff --git a/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc b/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc
index 16016d1cf8171..a34308aa93887 100644
--- a/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc
@@ -13,10 +13,9 @@ part of a full metric aggregation.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-`params['_agg']` (`Map`)::
+`state` (`Map`)::
         `Map` used to add values for processing in a
-        <<painless-metric-agg-map-context, combine script>> or returned
-        directly.
+        <<painless-metric-agg-map-context, combine script>> or to be returned from the aggregation.
 
 `doc` (`Map`, read-only)::
         Contains the fields of the current document where each field is a
@@ -27,15 +26,16 @@ part of a full metric aggregation.
 
 *Side Effects*
 
-`params['_agg']` (`Map`)::
+`state` (`Map`)::
         Use this `Map` to add values for processing in a combine script.
         Additional values must be of the type `Map`, `List`, `String` or
-        primitive. If an initialization script is provided as part the
+        primitive. The same `state` `Map` is shared between all aggregated documents
+        on a given shard. If an initialization script is provided as part of the
         aggregation then values added from the initialization script are
-        available as well.  If no combine script is specified, values must be
-        directly stored in `_agg`. If no combine script and no
+        available.  If no combine script is specified, values must be
+        directly stored in `state` in a usable form. If no combine script and no
         <<painless-metric-agg-reduce-context, reduce script>> are specified, the
-        values are used as the result.
+        `state` values are used as the result.
 
 *Return*
 
diff --git a/docs/painless/painless-contexts/painless-metric-agg-reduce-context.asciidoc b/docs/painless/painless-contexts/painless-metric-agg-reduce-context.asciidoc
index b76e02b1b0499..b492207ef4468 100644
--- a/docs/painless/painless-contexts/painless-metric-agg-reduce-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-metric-agg-reduce-context.asciidoc
@@ -14,7 +14,7 @@ specified) and is optional as part of a full metric aggregation.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-`params['_aggs']` (`Map`)::
+`states` (`Map`)::
         `Map` with values available from the prior combine script (or a map
         script if no combine script is specified).
 
diff --git a/docs/painless/painless-getting-started.asciidoc b/docs/painless/painless-getting-started.asciidoc
index 1dec4a33bb583..8cff207ab04d5 100644
--- a/docs/painless/painless-getting-started.asciidoc
+++ b/docs/painless/painless-getting-started.asciidoc
@@ -198,7 +198,7 @@ POST hockey/player/1/_update
 ==== Dates
 
 Date fields are exposed as
-`ReadableDateTime`
+`ReadableDateTime` or
 so they support methods like
 `getYear`,
 and `getDayOfWeek`.
@@ -220,6 +220,11 @@ GET hockey/_search
 }
 ----------------------------------------------------------------
 // CONSOLE
+// TEST[warning:The joda time api for doc values is deprecated. Use -Des.scripting.use_java_time=true to use the java time api for date field doc values]
+
+NOTE: Date fields are changing in 7.0 to be exposed as `ZonedDateTime`
+from Java 8's time API. To switch to this functionality early, 
+add `-Des.scripting.use_java_time=true` to `jvm.options`.
 
 [float]
 [[modules-scripting-painless-regex]]
diff --git a/docs/painless/painless-keywords.asciidoc b/docs/painless/painless-keywords.asciidoc
index 9463902c8d346..24371d3713c0b 100644
--- a/docs/painless/painless-keywords.asciidoc
+++ b/docs/painless/painless-keywords.asciidoc
@@ -5,7 +5,7 @@ Keywords are reserved tokens for built-in language features.
 
 *Errors*
 
-If a keyword is used as an <<painless-identifiers, identifier>>.
+* If a keyword is used as an <<painless-identifiers, identifier>>.
 
 *Keywords*
 
diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc
index 5cdc568e16250..f79e73fcf55db 100644
--- a/docs/plugins/discovery-ec2.asciidoc
+++ b/docs/plugins/discovery-ec2.asciidoc
@@ -37,7 +37,6 @@ bin/elasticsearch-keystore add discovery.ec2.secret_key
 The following are the available discovery settings. All should be prefixed with `discovery.ec2.`.
 Those that must be stored in the keystore are marked as `Secure`.
 
-
 `access_key`::
 
     An ec2 access key. The `secret_key` setting must also be specified. (Secure)
@@ -122,6 +121,10 @@ Defaults to `private_ip`.
     How long the list of hosts is cached to prevent further requests to the AWS API.
     Defaults to `10s`.
 
+*All* secure settings of this plugin are {ref}/secure-settings.html#reloadable-secure-settings[reloadable].
+After you reload the settings, an aws sdk client with the latest settings
+from the keystore will be used.
+
 [IMPORTANT]
 .Binding the network host
 ==============================================
diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc
index 893acf05a4bad..13c6a7b62ccbd 100644
--- a/docs/plugins/repository-azure.asciidoc
+++ b/docs/plugins/repository-azure.asciidoc
@@ -11,7 +11,7 @@ include::install_remove.asciidoc[]
 ==== Azure Repository
 
 To enable Azure repositories, you have first to define your azure storage settings as
-{ref}/secure-settings.html[secured settings]:
+{ref}/secure-settings.html[secure settings], before starting up the node:
 
 [source,sh]
 ----------------------------------------------------------------
@@ -20,6 +20,7 @@ bin/elasticsearch-keystore add azure.client.default.key
 ----------------------------------------------------------------
 
 Where `account` is the azure account name and `key` the azure secret key.
+These settings are used by the repository's internal azure client.
 
 Note that you can also define more than one account:
 
@@ -31,7 +32,18 @@ bin/elasticsearch-keystore add azure.client.secondary.account
 bin/elasticsearch-keystore add azure.client.secondary.key
 ----------------------------------------------------------------
 
-`default` is the default account name which will be used by a repository unless you set an explicit one.
+`default` is the default account name which will be used by a repository,
+unless you set an explicit one in the
+<<repository-azure-repository-settings, repository settings>>.
+
+Both `account` and `key` storage settings are
+{ref}/secure-settings.html#reloadable-secure-settings[reloadable]. After you
+reload the settings, the internal azure clients, which are used to transfer the
+snapshot, will utilize the latest settings from the keystore.
+
+NOTE: In progress snapshot/restore jobs will not be preempted by a *reload*
+of the storage secure settings. They will complete using the client as it was built
+when the operation started.
 
 You can set the client side timeout to use when making any single request. It can be defined globally, per account or both.
 It's not set by default which means that Elasticsearch is using the
diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc
index 84fb47d1761e4..8292325738506 100644
--- a/docs/plugins/repository-gcs.asciidoc
+++ b/docs/plugins/repository-gcs.asciidoc
@@ -112,6 +112,15 @@ PUT _snapshot/my_gcs_repository
 // CONSOLE
 // TEST[skip:we don't have gcs setup while testing this]
 
+The `credentials_file` settings are {ref}/secure-settings.html#reloadable-secure-settings[reloadable].
+After you reload the settings, the internal `gcs` clients, used to transfer the
+snapshot contents, will utilize the latest settings from the keystore.
+
+
+NOTE: In progress snapshot/restore jobs will not be preempted by a *reload*
+of the client's `credentials_file` settings. They will complete using the client
+as it was built when the operation started.
+
 [[repository-gcs-client]]
 ==== Client Settings
 
diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index 19ead367204ba..b065951856a45 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -35,9 +35,9 @@ PUT _snapshot/my_s3_repository
 ==== Client Settings
 
 The client used to connect to S3 has a number of settings available. Client setting names are of
-the form `s3.client.CLIENT_NAME.SETTING_NAME` and specified inside `elasticsearch.yml`.  The
-default client name looked up by a `s3` repository is called `default`, but can be customized
-with the repository setting `client`. For example:
+the form `s3.client.CLIENT_NAME.SETTING_NAME`. The default client name, which is looked up by
+an `s3` repository, is called `default`. It can be modified using the
+<<repository-s3-repository, repository setting>> `client`. For example:
 
 [source,js]
 ----
@@ -53,8 +53,11 @@ PUT _snapshot/my_s3_repository
 // CONSOLE
 // TEST[skip:we don't have s3 setup while testing this]
 
-Some settings are sensitive and must be stored in the {ref}/secure-settings.html[elasticsearch keystore].
-For example, to use explicit AWS access keys:
+Most client settings are specified inside `elasticsearch.yml`, but some are
+sensitive and must be stored in the {ref}/secure-settings.html[elasticsearch keystore].
+
+For example, before you start the node, run these commands to add AWS access
+key settings to the keystore:
 
 [source,sh]
 ----
@@ -62,8 +65,19 @@ bin/elasticsearch-keystore add s3.client.default.access_key
 bin/elasticsearch-keystore add s3.client.default.secret_key
 ----
 
-The following are the available client settings. Those that must be stored in the keystore
-are marked as `Secure`.
+*All* client secure settings of this plugin are
+{ref}/secure-settings.html#reloadable-secure-settings[reloadable]. After you
+reload the settings, the internal `s3` clients, used to transfer the snapshot
+contents, will utilize the latest settings from the keystore. Any existing `s3`
+repositories, as well as any newly created ones, will pick up the new values
+stored in the keystore.
+
+NOTE: In progress snapshot/restore tasks will not be preempted by a *reload*
+of the client's secure settings. The task will complete using the client as it
+was built when the operation started.
+
+The following is the list of all the available client settings.
+Those that must be stored in the keystore are marked as `Secure` and are *reloadable*.
 
 `access_key`::
 
diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
index efbd8ef7389bb..e19ecac462d14 100644
--- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
@@ -425,6 +425,7 @@ POST /sales/_search?size=0
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:sales]
+// TEST[warning:The joda time api for doc values is deprecated. Use -Des.scripting.use_java_time=true to use the java time api for date field doc values]
 
 Response:
 
diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index 551c082a6629e..f42d176aea06c 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -653,7 +653,7 @@ GET /_search
 // CONSOLE
 
 In the above example, buckets will be created for all the tags that has the word `sport` in them, except those starting
-with `water_` (so the tag `water_sports` will no be aggregated). The `include` regular expression will determine what
+with `water_` (so the tag `water_sports` will not be aggregated). The `include` regular expression will determine what
 values are "allowed" to be aggregated, while the `exclude` determines the values that should not be aggregated. When
 both are defined, the `exclude` has precedence, meaning, the `include` is evaluated first and only then the `exclude`.
 
diff --git a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
index 3da1c60db0552..a998e6b11f9ab 100644
--- a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
+++ b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
@@ -87,7 +87,7 @@ The above example produces the following term:
 
 [source,text]
 ---------------------------
-[ My, credit, card, is 123_456_789 ]
+[ My, credit, card, is, 123_456_789 ]
 ---------------------------
 
 WARNING: Using a replacement string that changes the length of the original
diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc
index 191da2660d668..3de850418719d 100644
--- a/docs/reference/cluster/stats.asciidoc
+++ b/docs/reference/cluster/stats.asciidoc
@@ -22,6 +22,7 @@ Will return, for example:
       "successful" : 1,
       "failed" : 0
    },
+   "cluster_uuid": "YjAvIhsCQ9CbjWZb2qJw3Q",
    "cluster_name": "elasticsearch",
    "timestamp": 1459427693515,
    "status": "green",
diff --git a/docs/reference/index-modules/slowlog.asciidoc b/docs/reference/index-modules/slowlog.asciidoc
index 1dc84f380f509..235256bdce7c0 100644
--- a/docs/reference/index-modules/slowlog.asciidoc
+++ b/docs/reference/index-modules/slowlog.asciidoc
@@ -50,7 +50,7 @@ appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
 appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
 appender.index_search_slowlog_rolling.layout.type = PatternLayout
-appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %.-10000m%n
 appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%i.log.gz
 appender.index_search_slowlog_rolling.policies.type = Policies
 appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy
@@ -103,7 +103,7 @@ appender.index_indexing_slowlog_rolling.type = RollingFile
 appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
 appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
 appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
-appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %.-10000m%n
 appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%i.log.gz
 appender.index_indexing_slowlog_rolling.policies.type = Policies
 appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy
diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc
index 8e0e84c6657be..81a96510e46cd 100644
--- a/docs/reference/indices/aliases.asciidoc
+++ b/docs/reference/indices/aliases.asciidoc
@@ -257,6 +257,9 @@ and there are multiple indices referenced by an alias, then writes will not be a
 It is possible to specify an index associated with an alias as a write index using both the aliases API
 and index creation API.
 
+Setting an index to be the write index with an alias also affects how the alias is manipulated during
+Rollover (see <<indices-rollover-index, Rollover With Write Index>>).
+
 [source,js]
 --------------------------------------------------
 POST /_aliases
diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index 1e3a361f1b14e..1730c95e0dd24 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -4,10 +4,19 @@
 The rollover index API rolls an alias over to a new index when the existing
 index is considered to be too large or too old.
 
-The API accepts a single alias name and a list of `conditions`.  The alias
-must point to a single index only.  If the index satisfies the specified
-conditions then a new index is created and the alias is switched to point to
-the new index.
+The API accepts a single alias name and a list of `conditions`. The alias must point to a write index for
+a Rollover request to be valid. There are two ways this can be achieved, and depending on the configuration, the
+alias metadata will be updated differently. The two scenarios are as follows:
+
+ - The alias only points to a single index with `is_write_index` not configured (defaults to `null`).
+
+In this scenario, the original index will have their rollover alias will be added to the newly created index, and removed
+from the original (rolled-over) index.
+
+ - The alias points to one or more indices with `is_write_index` set to `true` on the index to be rolled over (the write index).
+
+In this scenario, the write index will have its rollover alias' `is_write_index` set to `false`, while the newly created index
+will now have the rollover alias pointing to it as the write index with `is_write_index` as `true`.
 
 
 [source,js]
@@ -231,3 +240,98 @@ POST /logs_write/_rollover?dry_run
 Because the rollover operation creates a new index to rollover to, the
 <<create-index-wait-for-active-shards,`wait_for_active_shards`>> setting on
 index creation applies to the rollover action as well.
+
+[[indices-rollover-is-write-index]]
+[float]
+=== Write Index Alias Behavior
+
+The rollover alias when rolling over a write index that has `is_write_index` explicitly set to `true` is not
+swapped during rollover actions. Since having an alias point to multiple indices is ambiguous in distinguishing
+which is the correct write index to roll over, it is not valid to rollover an alias that points to multiple indices.
+For this reason, the default behavior is to swap which index is being pointed to by the write-oriented alias. This
+was `logs_write` in some of the above examples. Since setting `is_write_index` enables an alias to point to multiple indices
+while also being explicit as to which is the write index that rollover should target, removing the alias from the rolled over
+index is not necessary. This simplifies things by allowing for one alias to behave both as the write and read aliases for
+indices that are being managed with Rollover.
+
+Look at the behavior of the aliases in the following example where `is_write_index` is set on the rolled over index.
+
+[source,js]
+--------------------------------------------------
+PUT my_logs_index-000001
+{
+  "aliases": {
+    "logs": { "is_write_index": true } <1>
+  }
+}
+
+PUT logs/_doc/1
+{
+  "message": "a dummy log"
+}
+
+POST logs/_refresh
+
+POST /logs/_rollover
+{
+  "conditions": {
+    "max_docs":   "1"
+  }
+}
+
+PUT logs/_doc/2 <2>
+{
+  "message": "a newer log"
+}
+--------------------------------------------------
+// CONSOLE
+<1> configures `my_logs_index` as the write index for the `logs` alias
+<2> newly indexed documents against the `logs` alias will write to the new index
+
+[source,js]
+--------------------------------------------------
+{
+  "_index" : "my_logs_index-000002",
+  "_type" : "_doc",
+  "_id" : "2",
+  "_version" : 1,
+  "result" : "created",
+  "_shards" : {
+    "total" : 2,
+    "successful" : 1,
+    "failed" : 0
+  },
+  "_seq_no" : 0,
+  "_primary_term" : 1
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+//////////////////////////
+[source,js]
+--------------------------------------------------
+GET _alias
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+//////////////////////////
+
+After the rollover, the alias metadata for the two indices will have the `is_write_index` setting
+reflect each index's role, with the newly created index as the write index.
+
+[source,js]
+--------------------------------------------------
+{
+  "my_logs_index-000002": {
+    "aliases": {
+      "logs": { "is_write_index": true }
+    }
+  },
+  "my_logs_index-000001": {
+    "aliases": {
+      "logs": { "is_write_index" : false }
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 79277d22e8128..37c616b234991 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -1049,6 +1049,125 @@ understands this to mean `2016-04-01` as is explained in the <<date-math-index-n
 | `index_name_format`    | no        | yyyy-MM-dd                   | The format to be used when printing the parsed date into the index name. An valid Joda pattern is expected here.
 |======
 
+[[dot-expand-processor]]
+=== Dot Expander Processor
+
+Expands a field with dots into an object field. This processor allows fields
+with dots in the name to be accessible by other processors in the pipeline.
+Otherwise these <<accessing-data-in-pipelines,fields>> can't be accessed by any processor.
+
+[[dot-expender-options]]
+.Dot Expand Options
+[options="header"]
+|======
+| Name     | Required  | Default  | Description
+| `field`  | yes       | -        | The field to expand into an object field
+| `path`   | no        | -        | The field that contains the field to expand. Only required if the field to expand is part another object field, because the `field` option can only understand leaf fields.
+|======
+
+[source,js]
+--------------------------------------------------
+{
+  "dot_expander": {
+    "field": "foo.bar"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+For example the dot expand processor would turn this document:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo.bar" : "value"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+into:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo" : {
+    "bar" : "value"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+If there is already a `bar` field nested under `foo` then
+this processor merges the `foo.bar` field into it. If the field is
+a scalar value then it will turn that field into an array field.
+
+For example, the following document:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo.bar" : "value2",
+  "foo" : {
+    "bar" : "value1"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+is transformed by the `dot_expander` processor into:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo" : {
+    "bar" : ["value1", "value2"]
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+If any field outside of the leaf field conflicts with a pre-existing field of the same name,
+then that field needs to be renamed first.
+
+Consider the following document:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo": "value1",
+  "foo.bar": "value2"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+Then the `foo` needs to be renamed first before the `dot_expander`
+processor is applied. So in order for the `foo.bar` field to properly
+be expanded into the `bar` field under the `foo` field the following
+pipeline should be used:
+
+[source,js]
+--------------------------------------------------
+{
+  "processors" : [
+    {
+      "rename" : {
+        "field" : "foo",
+        "target_field" : "foo.bar""
+      }
+    },
+    {
+      "dot_expander": {
+        "field": "foo.bar"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+The reason for this is that Ingest doesn't know how to automatically cast
+a scalar field to an object field.
+
 [[fail-processor]]
 === Fail Processor
 Raises an exception. This is useful for when
@@ -2058,125 +2177,6 @@ Converts a string to its uppercase equivalent.
 --------------------------------------------------
 // NOTCONSOLE
 
-[[dot-expand-processor]]
-=== Dot Expander Processor
-
-Expands a field with dots into an object field. This processor allows fields
-with dots in the name to be accessible by other processors in the pipeline.
-Otherwise these <<accessing-data-in-pipelines,fields>> can't be accessed by any processor.
-
-[[dot-expender-options]]
-.Dot Expand Options
-[options="header"]
-|======
-| Name     | Required  | Default  | Description
-| `field`  | yes       | -        | The field to expand into an object field
-| `path`   | no        | -        | The field that contains the field to expand. Only required if the field to expand is part another object field, because the `field` option can only understand leaf fields.
-|======
-
-[source,js]
---------------------------------------------------
-{
-  "dot_expander": {
-    "field": "foo.bar"
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-For example the dot expand processor would turn this document:
-
-[source,js]
---------------------------------------------------
-{
-  "foo.bar" : "value"
-}
---------------------------------------------------
-// NOTCONSOLE
-
-into:
-
-[source,js]
---------------------------------------------------
-{
-  "foo" : {
-    "bar" : "value"
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-If there is already a `bar` field nested under `foo` then
-this processor merges the `foo.bar` field into it. If the field is
-a scalar value then it will turn that field into an array field.
-
-For example, the following document:
-
-[source,js]
---------------------------------------------------
-{
-  "foo.bar" : "value2",
-  "foo" : {
-    "bar" : "value1"
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-is transformed by the `dot_expander` processor into:
-
-[source,js]
---------------------------------------------------
-{
-  "foo" : {
-    "bar" : ["value1", "value2"]
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-If any field outside of the leaf field conflicts with a pre-existing field of the same name,
-then that field needs to be renamed first.
-
-Consider the following document:
-
-[source,js]
---------------------------------------------------
-{
-  "foo": "value1",
-  "foo.bar": "value2"
-}
---------------------------------------------------
-// NOTCONSOLE
-
-Then the `foo` needs to be renamed first before the `dot_expander`
-processor is applied. So in order for the `foo.bar` field to properly
-be expanded into the `bar` field under the `foo` field the following
-pipeline should be used:
-
-[source,js]
---------------------------------------------------
-{
-  "processors" : [
-    {
-      "rename" : {
-        "field" : "foo",
-        "target_field" : "foo.bar""
-      }
-    },
-    {
-      "dot_expander": {
-        "field": "foo.bar"
-      }
-    }
-  ]
-}
---------------------------------------------------
-// NOTCONSOLE
-
-The reason for this is that Ingest doesn't know how to automatically cast
-a scalar field to an object field.
-
 [[urldecode-processor]]
 === URL Decode Processor
 URL-decodes a string
diff --git a/docs/reference/query-dsl/fuzzy-query.asciidoc b/docs/reference/query-dsl/fuzzy-query.asciidoc
index fa11cb257d4b4..4be546916240f 100644
--- a/docs/reference/query-dsl/fuzzy-query.asciidoc
+++ b/docs/reference/query-dsl/fuzzy-query.asciidoc
@@ -5,10 +5,10 @@ The fuzzy query uses similarity based on Levenshtein edit distance.
 
 ==== String fields
 
-The `fuzzy` query generates all possible matching terms that are within  the
+The `fuzzy` query generates matching terms that are within the
 maximum edit distance specified in `fuzziness` and then checks the term
 dictionary to find out which of those generated terms actually exist in the
-index.
+index. The final query uses up to `max_expansions` matching terms.
 
 Here is a simple example:
 
diff --git a/docs/reference/release-notes/7.0.0-alpha1.asciidoc b/docs/reference/release-notes/7.0.0-alpha1.asciidoc
index cf2e1e30be050..c3a03d77f8118 100644
--- a/docs/reference/release-notes/7.0.0-alpha1.asciidoc
+++ b/docs/reference/release-notes/7.0.0-alpha1.asciidoc
@@ -21,4 +21,10 @@ Aggregations::
 * The Percentiles and PercentileRanks aggregations now return `null` in the REST response,
   instead of `NaN`.  This makes it consistent with the rest of the aggregations.  Note:
   this only applies to the REST response, the java objects continue to return `NaN` (also
-  consistent with other aggregations)
\ No newline at end of file
+  consistent with other aggregations)
+
+Suggesters::
+* Plugins that register suggesters can now define their own types of suggestions and must
+  explicitly indicate the type of suggestion that they produce. Existing plugins will
+  require changes to their plugin registration. See the `custom-suggester` example
+  plugin {pull}30284[#30284]
\ No newline at end of file
diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc
index b9b99b708031e..8a9b59480a09b 100644
--- a/docs/reference/setup/important-settings.asciidoc
+++ b/docs/reference/setup/important-settings.asciidoc
@@ -14,6 +14,7 @@ The following settings *must* be considered before going to production:
 * <<heap-size,Heap size>>
 * <<heap-dump-path,Heap dump path>>
 * <<gc-logging,GC logging>>
+* <<es-tmpdir,Temp directory>>
 
 include::important-settings/path-settings.asciidoc[]
 
@@ -31,4 +32,6 @@ include::important-settings/heap-dump-path.asciidoc[]
 
 include::important-settings/gc-logging.asciidoc[]
 
+include::important-settings/es-tmpdir.asciidoc[]
+
 include::important-settings/error-file.asciidoc[]
diff --git a/docs/reference/setup/important-settings/es-tmpdir.asciidoc b/docs/reference/setup/important-settings/es-tmpdir.asciidoc
new file mode 100644
index 0000000000000..20959d969b879
--- /dev/null
+++ b/docs/reference/setup/important-settings/es-tmpdir.asciidoc
@@ -0,0 +1,23 @@
+[[es-tmpdir]]
+=== Temp directory
+
+By default, Elasticsearch uses a private temporary directory that the startup
+script creates immediately below the system temporary directory.
+
+On some Linux distributions a system utility will clean files and directories
+from `/tmp` if they have not been recently accessed. This can lead to the
+private temporary directory being removed while Elasticsearch is running if
+features that require the temporary directory are not used for a long time.
+This causes problems if a feature that requires the temporary directory is
+subsequently used.
+
+If you install Elasticsearch using the `.deb` or `.rpm` packages and run it
+under `systemd` then the private temporary directory that Elasticsearch uses
+is excluded from periodic cleanup.
+
+However, if you intend to run the `.tar.gz` distribution on Linux for an
+extended period then you should consider creating a dedicated temporary
+directory for Elasticsearch that is not under a path that will have old files
+and directories cleaned from it. This directory should have permissions set
+so that only the user that Elasticsearch runs as can access it. Then set the
+`$ES_TMPDIR` environment variable to point to it before starting Elasticsearch.
diff --git a/docs/reference/setup/important-settings/node-name.asciidoc b/docs/reference/setup/important-settings/node-name.asciidoc
index fab7ddcf11898..5980d8e284e1c 100644
--- a/docs/reference/setup/important-settings/node-name.asciidoc
+++ b/docs/reference/setup/important-settings/node-name.asciidoc
@@ -2,7 +2,7 @@
 === `node.name`
 
 By default, Elasticsearch will use the first seven characters of the randomly
-generated UUID as the node id.Note that the node id is persisted and does
+generated UUID as the node id. Note that the node id is persisted and does
 not change when a node restarts and therefore the default node name will also
 not change.
 
@@ -19,4 +19,4 @@ The `node.name` can also be set to the server's HOSTNAME as follows:
 [source,yaml]
 --------------------------------------------------
 node.name: ${HOSTNAME}
---------------------------------------------------
\ No newline at end of file
+--------------------------------------------------
diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc
index 523217b921a43..e2e5c6ab70b62 100644
--- a/docs/reference/setup/install/docker.asciidoc
+++ b/docs/reference/setup/install/docker.asciidoc
@@ -4,9 +4,9 @@
 {es} is also available as Docker images.
 The images use https://hub.docker.com/_/centos/[centos:7] as the base image.
 
-A list of all published Docker images and tags can be found in
-https://www.docker.elastic.co[www.docker.elastic.co]. The source code can be found
-on https://github.com/elastic/elasticsearch-docker/tree/{branch}[GitHub].
+A list of all published Docker images and tags is available at
+https://www.docker.elastic.co[www.docker.elastic.co]. The source code is in
+https://github.com/elastic/elasticsearch-docker/tree/{branch}[GitHub].
 
 These images are free to use under the Elastic license. They contain open source 
 and free commercial features and access to paid commercial features.  
@@ -29,15 +29,13 @@ endif::[]
 
 ifeval::["{release-state}"!="unreleased"]
 
-For example, the Docker image can be retrieved with the following command:
-
 ["source","sh",subs="attributes"]
 --------------------------------------------
 docker pull {docker-repo}:{version}
 --------------------------------------------
 
 Alternatively, you can download other Docker images that contain only features 
-that are available under the Apache 2.0 license from 
+available under the Apache 2.0 license. To download the images, go to 
 https://www.docker.elastic.co[www.docker.elastic.co]. 
 
 endif::[]
diff --git a/docs/reference/setup/logging-config.asciidoc b/docs/reference/setup/logging-config.asciidoc
index d35b3db2df7f3..ac949bc941a9c 100644
--- a/docs/reference/setup/logging-config.asciidoc
+++ b/docs/reference/setup/logging-config.asciidoc
@@ -25,7 +25,7 @@ appender.rolling.type = RollingFile <1>
 appender.rolling.name = rolling
 appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log <2>
 appender.rolling.layout.type = PatternLayout
-appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %.-10000m%n
 appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz <3>
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy <4>
diff --git a/docs/reference/setup/secure-settings.asciidoc b/docs/reference/setup/secure-settings.asciidoc
index 1474ba137697d..2177440457acf 100644
--- a/docs/reference/setup/secure-settings.asciidoc
+++ b/docs/reference/setup/secure-settings.asciidoc
@@ -75,3 +75,34 @@ To remove a setting from the keystore, use the `remove` command:
 bin/elasticsearch-keystore remove the.setting.name.to.remove
 ----------------------------------------------------------------
 
+[float]
+[[reloadable-secure-settings]]
+=== Reloadable secure settings
+
+Just like the settings values in `elasticsearch.yml`, changes to the
+keystore contents are not automatically applied to the running
+elasticsearch node. Re-reading settings requires a node restart.
+However, certain secure settings are marked as *reloadable*. Such settings
+can be re-read and applied on a running node.
+
+The values of all secure settings, *reloadable* or not, must be identical
+across all cluster nodes. After making the desired secure settings changes,
+using the `bin/elasticsearch-keystore add` command, call:
+[source,js]
+----
+POST _nodes/reload_secure_settings
+{
+  "secure_settings_password": ""
+}
+----
+// CONSOLE
+This API will decrypt and re-read the entire keystore, on every cluster node,
+but only the *reloadable* secure settings will be applied. Changes to other
+settings will not go into effect until the next restart. Once the call returns,
+the reload has been completed, meaning that all internal datastructures dependent
+on these settings have been changed. Everything should look as if the settings
+had the new value from the start.
+
+When changing multiple *reloadable* secure settings, modify all of them, on
+each cluster node, and then issue a `reload_secure_settings` call, instead
+of reloading after each modification.
diff --git a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
index a8dd91e8b6de2..dbda453e5f9fb 100644
--- a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
+++ b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
@@ -92,8 +92,7 @@ protected ClientYamlTestClient initClientYamlTestClient(
             final List<HttpHost> hosts,
             final Version esVersion,
             final Version masterVersion) {
-        return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, masterVersion,
-                restClientBuilder -> configureClient(restClientBuilder, restClientSettings()));
+        return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, masterVersion, this::getClientBuilderWithSniffedHosts);
     }
 
     /**
diff --git a/libs/grok/src/main/java/org/elasticsearch/grok/ThreadWatchdog.java b/libs/grok/src/main/java/org/elasticsearch/grok/ThreadWatchdog.java
index d0de7637d2c08..f3515fcfe83b0 100644
--- a/libs/grok/src/main/java/org/elasticsearch/grok/ThreadWatchdog.java
+++ b/libs/grok/src/main/java/org/elasticsearch/grok/ThreadWatchdog.java
@@ -21,6 +21,8 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiFunction;
 import java.util.function.LongSupplier;
 
@@ -104,6 +106,8 @@ class Default implements ThreadWatchdog {
         private final long maxExecutionTime;
         private final LongSupplier relativeTimeSupplier;
         private final BiFunction<Long, Runnable, ScheduledFuture<?>> scheduler;
+        private final AtomicInteger registered = new AtomicInteger(0);
+        private final AtomicBoolean running = new AtomicBoolean(false);
         final ConcurrentHashMap<Thread, Long> registry = new ConcurrentHashMap<>();
         
         private Default(long interval,
@@ -114,11 +118,14 @@ private Default(long interval,
             this.maxExecutionTime = maxExecutionTime;
             this.relativeTimeSupplier = relativeTimeSupplier;
             this.scheduler = scheduler;
-            scheduler.apply(interval, this::interruptLongRunningExecutions);
         }
         
         public void register() {
+            registered.getAndIncrement();
             Long previousValue = registry.put(Thread.currentThread(), relativeTimeSupplier.getAsLong());
+            if (running.compareAndSet(false, true) == true) {
+                scheduler.apply(interval, this::interruptLongRunningExecutions);
+            }
             assert previousValue == null;
         }
     
@@ -129,6 +136,7 @@ public long maxExecutionTimeInMillis() {
     
         public void unregister() {
             Long previousValue = registry.remove(Thread.currentThread());
+            registered.decrementAndGet();
             assert previousValue != null;
         }
         
@@ -140,7 +148,11 @@ private void interruptLongRunningExecutions() {
                     // not removing the entry here, this happens in the unregister() method.
                 }
             }
-            scheduler.apply(interval, this::interruptLongRunningExecutions);
+            if (registered.get() > 0) {
+                scheduler.apply(interval, this::interruptLongRunningExecutions);
+            } else {
+                running.set(false);
+            }
         }
         
     }
diff --git a/libs/grok/src/test/java/org/elasticsearch/grok/ThreadWatchdogTests.java b/libs/grok/src/test/java/org/elasticsearch/grok/ThreadWatchdogTests.java
index 46faa4ae05d38..29e2351215f60 100644
--- a/libs/grok/src/test/java/org/elasticsearch/grok/ThreadWatchdogTests.java
+++ b/libs/grok/src/test/java/org/elasticsearch/grok/ThreadWatchdogTests.java
@@ -18,15 +18,25 @@
  */
 package org.elasticsearch.grok;
 
-import org.elasticsearch.test.ESTestCase;
-
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import org.elasticsearch.test.ESTestCase;
+import org.mockito.Mockito;
 
 import static org.hamcrest.Matchers.is;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
 
 public class ThreadWatchdogTests extends ESTestCase {
-    
+
     public void testInterrupt() throws Exception {
         AtomicBoolean run = new AtomicBoolean(true); // to avoid a lingering thread when test has completed
         ThreadWatchdog watchdog = ThreadWatchdog.newInstance(10, 100, System::currentTimeMillis, (delay, command) -> {
@@ -43,7 +53,7 @@ public void testInterrupt() throws Exception {
             thread.start();
             return null;
         });
-    
+
         Map<?, ?> registry = ((ThreadWatchdog.Default) watchdog).registry;
         assertThat(registry.size(), is(0));
         // need to call #register() method on a different thread, assertBusy() fails if current thread gets interrupted
@@ -66,5 +76,39 @@ public void testInterrupt() throws Exception {
             assertThat(registry.size(), is(0));
         });
     }
-    
+
+    public void testIdleIfNothingRegistered() throws Exception {
+        long interval = 1L;
+        ScheduledExecutorService threadPool = mock(ScheduledExecutorService.class);
+        ThreadWatchdog watchdog = ThreadWatchdog.newInstance(interval, Long.MAX_VALUE, System::currentTimeMillis,
+            (delay, command) -> threadPool.schedule(command, delay, TimeUnit.MILLISECONDS));
+        // Periodic action is not scheduled because no thread is registered
+        verifyZeroInteractions(threadPool);
+        CompletableFuture<Runnable> commandFuture = new CompletableFuture<>();
+        // Periodic action is scheduled because a thread is registered
+        doAnswer(invocationOnMock -> {
+            commandFuture.complete((Runnable) invocationOnMock.getArguments()[0]);
+            return null;
+        }).when(threadPool).schedule(
+            any(Runnable.class), eq(interval), eq(TimeUnit.MILLISECONDS)
+        );
+        watchdog.register();
+        // Registering the first thread should have caused the command to get scheduled again
+        Runnable command = commandFuture.get(1L, TimeUnit.MILLISECONDS);
+        Mockito.reset(threadPool);
+        watchdog.unregister();
+        command.run();
+        // Periodic action is not scheduled again because no thread is registered
+        verifyZeroInteractions(threadPool);
+        watchdog.register();
+        Thread otherThread = new Thread(watchdog::register);
+        try {
+            verify(threadPool).schedule(any(Runnable.class), eq(interval), eq(TimeUnit.MILLISECONDS));
+            // Registering a second thread does not cause the command to get scheduled twice
+            verifyNoMoreInteractions(threadPool);
+            otherThread.start();
+        } finally {
+            otherThread.join();
+        }
+    }
 }
diff --git a/libs/x-content/licenses/jackson-core-2.8.10.jar.sha1 b/libs/x-content/licenses/jackson-core-2.8.10.jar.sha1
deleted file mode 100644
index a322d371e265e..0000000000000
--- a/libs/x-content/licenses/jackson-core-2.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-eb21a035c66ad307e66ec8fce37f5d50fd62d039
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.8.10.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.8.10.jar.sha1
deleted file mode 100644
index 1d3e18e21a694..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-cbor-2.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1c58cc9313ddf19f0900cd61ed044874278ce320
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..378ba524422bc
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
@@ -0,0 +1 @@
+8b9826e16c3366764bfb7ad7362554f0471046c3
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.8.10.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.8.10.jar.sha1
deleted file mode 100644
index 4f4cacde22079..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-smile-2.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e853081fadaad3e98ed801937acc3d8f77580686
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..510afb3df53e6
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1
@@ -0,0 +1 @@
+d9d1c49c5d9d5e46e2aee55f3cdd119286fe0fc1
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.8.10.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.8.10.jar.sha1
deleted file mode 100644
index 40bcb05f69795..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-yaml-2.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1e08caf1d787c825307d8cc6362452086020d853
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..78a68d715ec3d
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1
@@ -0,0 +1 @@
+2e77c6ff7342cd61ab1ae7cb14ed16aebfc8a72a
\ No newline at end of file
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java
index fb871590df7fd..38bc251be41dd 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java
@@ -35,7 +35,7 @@
  */
 public class XContentFactory {
 
-    private static final int GUESS_HEADER_LENGTH = 20;
+    static final int GUESS_HEADER_LENGTH = 20;
 
     /**
      * Returns a content builder using JSON format ({@link org.elasticsearch.common.xcontent.XContentType#JSON}.
@@ -153,8 +153,10 @@ public static XContentType xContentType(CharSequence content) {
             return XContentType.JSON;
         }
         // Should we throw a failure here? Smile idea is to use it in bytes....
-        if (length > 2 && first == SmileConstants.HEADER_BYTE_1 && content.charAt(1) == SmileConstants.HEADER_BYTE_2 &&
-            content.charAt(2) == SmileConstants.HEADER_BYTE_3) {
+        if (length > 2
+                && first == SmileConstants.HEADER_BYTE_1
+                && content.charAt(1) == SmileConstants.HEADER_BYTE_2
+                && content.charAt(2) == SmileConstants.HEADER_BYTE_3) {
             return XContentType.SMILE;
         }
         if (length > 2 && first == '-' && content.charAt(1) == '-' && content.charAt(2) == '-') {
@@ -227,13 +229,29 @@ public static XContent xContent(byte[] data, int offset, int length) {
      */
     @Deprecated
     public static XContentType xContentType(InputStream si) throws IOException {
+        /*
+         * We need to guess the content type. To do this, we look for the first non-whitespace character and then try to guess the content
+         * type on the GUESS_HEADER_LENGTH bytes that follow. We do this in a way that does not modify the initial read position in the
+         * underlying input stream. This is why the input stream must support mark/reset and why we repeatedly mark the read position and
+         * reset.
+         */
         if (si.markSupported() == false) {
             throw new IllegalArgumentException("Cannot guess the xcontent type without mark/reset support on " + si.getClass());
         }
-        si.mark(GUESS_HEADER_LENGTH);
+        si.mark(Integer.MAX_VALUE);
         try {
+            // scan until we find the first non-whitespace character or the end of the stream
+            int current;
+            do {
+                current = si.read();
+                if (current == -1) {
+                    return null;
+                }
+            } while (Character.isWhitespace((char) current));
+            // now guess the content type off the next GUESS_HEADER_LENGTH bytes including the current byte
             final byte[] firstBytes = new byte[GUESS_HEADER_LENGTH];
-            int read = 0;
+            firstBytes[0] = (byte) current;
+            int read = 1;
             while (read < GUESS_HEADER_LENGTH) {
                 final int r = si.read(firstBytes, read, GUESS_HEADER_LENGTH - read);
                 if (r == -1) {
@@ -245,6 +263,7 @@ public static XContentType xContentType(InputStream si) throws IOException {
         } finally {
             si.reset();
         }
+
     }
 
     /**
@@ -278,15 +297,17 @@ public static XContentType xContentType(byte[] bytes, int offset, int length) {
         if (first == '{') {
             return XContentType.JSON;
         }
-        if (length > 2 && first == SmileConstants.HEADER_BYTE_1 && bytes[offset + 1] == SmileConstants.HEADER_BYTE_2 &&
-            bytes[offset + 2] == SmileConstants.HEADER_BYTE_3) {
+        if (length > 2
+                && first == SmileConstants.HEADER_BYTE_1
+                && bytes[offset + 1] == SmileConstants.HEADER_BYTE_2
+                && bytes[offset + 2] == SmileConstants.HEADER_BYTE_3) {
             return XContentType.SMILE;
         }
         if (length > 2 && first == '-' && bytes[offset + 1] == '-' && bytes[offset + 2] == '-') {
             return XContentType.YAML;
         }
         // CBOR logic similar to CBORFactory#hasCBORFormat
-        if (first == CBORConstants.BYTE_OBJECT_INDEFINITE && length > 1){
+        if (first == CBORConstants.BYTE_OBJECT_INDEFINITE && length > 1) {
             return XContentType.CBOR;
         }
         if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_TAG, first) && length > 2) {
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
index 7bb3ebfba6e36..88cba512b86cb 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
@@ -68,8 +68,7 @@ public void execute(IngestDocument ingestDocument) throws Exception {
             throw new IllegalArgumentException("Provided Grok expressions do not match field value: [" + fieldValue + "]");
         }
 
-        matches.entrySet().stream()
-            .forEach((e) -> ingestDocument.setFieldValue(e.getKey(), e.getValue()));
+        matches.forEach(ingestDocument::setFieldValue);
 
         if (traceMatch) {
             if (matchPatterns.size() > 1) {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
index 500f1aa1719c3..34fd4ec27a3b3 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
@@ -67,10 +67,9 @@ public void testConvertIntLeadingZero() throws Exception {
         assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(10));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32370")
     public void testConvertIntHexError() {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
-        String value = "0x" + randomAlphaOfLengthBetween(1, 10);
+        String value = "0xnotanumber";
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value);
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
@@ -138,10 +137,9 @@ public void testConvertLongLeadingZero() throws Exception {
         assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(10L));
     }
 
-    @AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/32370")
     public void testConvertLongHexError() {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
-        String value = "0x" + randomAlphaOfLengthBetween(1, 10);
+        String value = "0xnotanumber";
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value);
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml
new file mode 100644
index 0000000000000..c20d7698131bf
--- /dev/null
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml
@@ -0,0 +1,73 @@
+---
+teardown:
+  - do:
+      ingest.delete_pipeline:
+        id: "my_pipeline"
+        ignore: 404
+
+---
+"Test index with default pipeline":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "bytes" : {
+                  "field" : "bytes_source_field",
+                  "target_field" : "bytes_target_field"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              default_pipeline: "my_pipeline"
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        body: {bytes_source_field: "1kb"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.bytes_source_field: "1kb" }
+  - match: { _source.bytes_target_field: 1024 }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 2
+        pipeline: "_none"
+        body: {bytes_source_field: "1kb"}
+        
+  - do:
+      get:
+        index: test
+        type: test
+        id: 2
+  - match: { _source.bytes_source_field: "1kb" }
+  - is_false: _source.bytes_target_field
+  
+  - do:
+      catch:  bad_request
+      index:
+        index: test
+        type: test
+        id: 3
+        pipeline: ""
+        body: {bytes_source_field: "1kb"}
diff --git a/modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..0ebdddcc5f1b5
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+fded6bb485b8b01bb2a9280162fd14d4d3ce4510
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-608f0277b0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index 908f70131b39d..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bd7d8078a2d0ad11a24f54156cc015630c96858a
\ No newline at end of file
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
index 1cde9c258b4f1..23dc0fd276cbe 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
@@ -34,10 +34,12 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
-import org.elasticsearch.index.mapper.GeoPointFieldMapper.GeoPointFieldType;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.GeoPointFieldMapper.GeoPointFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.script.BucketAggregationScript;
+import org.elasticsearch.script.BucketAggregationSelectorScript;
 import org.elasticsearch.script.ClassPermission;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.FilterScript;
@@ -54,6 +56,7 @@
 import java.security.PrivilegedAction;
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -112,6 +115,17 @@ protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundE
         } else if (context.instanceClazz.equals(ExecutableScript.class)) {
             ExecutableScript.Factory factory = (p) -> new ExpressionExecutableScript(expr, p);
             return context.factoryClazz.cast(factory);
+        } else if (context.instanceClazz.equals(BucketAggregationScript.class)) {
+            return context.factoryClazz.cast(newBucketAggregationScriptFactory(expr));
+        } else if (context.instanceClazz.equals(BucketAggregationSelectorScript.class)) {
+            BucketAggregationScript.Factory factory = newBucketAggregationScriptFactory(expr);
+            BucketAggregationSelectorScript.Factory wrappedFactory = parameters -> new BucketAggregationSelectorScript(parameters) {
+                @Override
+                public boolean execute() {
+                    return factory.newInstance(getParams()).execute() == 1.0;
+                }
+            };
+            return context.factoryClazz.cast(wrappedFactory);
         } else if (context.instanceClazz.equals(FilterScript.class)) {
             FilterScript.Factory factory = (p, lookup) -> newFilterScript(expr, lookup, p);
             return context.factoryClazz.cast(factory);
@@ -122,6 +136,37 @@ protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundE
         throw new IllegalArgumentException("expression engine does not know how to handle script context [" + context.name + "]");
     }
 
+    private static BucketAggregationScript.Factory newBucketAggregationScriptFactory(Expression expr) {
+        return parameters -> {
+            ReplaceableConstDoubleValues[] functionValuesArray =
+                new ReplaceableConstDoubleValues[expr.variables.length];
+            Map<String, ReplaceableConstDoubleValues> functionValuesMap = new HashMap<>();
+            for (int i = 0; i < expr.variables.length; ++i) {
+                functionValuesArray[i] = new ReplaceableConstDoubleValues();
+                functionValuesMap.put(expr.variables[i], functionValuesArray[i]);
+            }
+            return new BucketAggregationScript(parameters) {
+                @Override
+                public Double execute() {
+                    getParams().forEach((name, value) -> {
+                        ReplaceableConstDoubleValues placeholder = functionValuesMap.get(name);
+                        if (placeholder == null) {
+                            throw new IllegalArgumentException("Error using " + expr + ". " +
+                                "The variable [" + name + "] does not exist in the executable expressions script.");
+                        } else if (value instanceof Number == false) {
+                            throw new IllegalArgumentException("Error using " + expr + ". " +
+                                "Executable expressions scripts can only process numbers." +
+                                "  The variable [" + name + "] is not a number.");
+                        } else {
+                            placeholder.setValue(((Number) value).doubleValue());
+                        }
+                    });
+                    return expr.evaluate(functionValuesArray);
+                }
+            };
+        };
+    }
+
     private SearchScript.LeafFactory newSearchScript(Expression expr, SearchLookup lookup, @Nullable Map<String, Object> vars) {
         MapperService mapper = lookup.doc().mapperService();
         // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings,
@@ -267,7 +312,7 @@ public void setDocument(int docid) {
             };
         };
     }
-    
+
     private ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLookup lookup, @Nullable Map<String, Object> vars) {
         SearchScript.LeafFactory searchLeafFactory = newSearchScript(expr, lookup, vars);
         return new ScoreScript.LeafFactory() {
@@ -284,17 +329,17 @@ public ScoreScript newInstance(LeafReaderContext ctx) throws IOException {
                     public double execute() {
                         return script.runAsDouble();
                     }
-    
+
                     @Override
                     public void setDocument(int docid) {
                         script.setDocument(docid);
                     }
-    
+
                     @Override
                     public void setScorer(Scorer scorer) {
                         script.setScorer(scorer);
                     }
-    
+
                     @Override
                     public double get_score() {
                         return script.getScore();
diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index fb1ea441a9dd5..ed4b1d631e064 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -17,8 +17,6 @@
  * under the License.
  */
 
-
-
 esplugin {
   description 'An easy, safe and fast scripting language for Elasticsearch'
   classname 'org.elasticsearch.painless.PainlessPlugin'
@@ -26,6 +24,8 @@ esplugin {
 
 integTestCluster {
   module project.project(':modules:mapper-extras')
+  systemProperty 'es.scripting.use_java_time', 'true'
+  systemProperty 'es.scripting.update.ctx_in_params', 'false'
 }
 
 dependencies {
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java
index 55b64b0420df1..c38325edd1424 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java
@@ -28,7 +28,7 @@
  * constructors, methods, and fields that can be used within a Painless script at both compile-time
  * and run-time.
  *
- * A whitelist consists of several pieces with {@link WhitelistClass}s as the top level.  Each
+ * A whitelist consists of several pieces with {@link WhitelistClass}s as the top level. Each
  * {@link WhitelistClass} will contain zero-to-many {@link WhitelistConstructor}s, {@link WhitelistMethod}s, and
  * {@link WhitelistField}s which are what will be available with a Painless script.  See each individual
  * whitelist object for more detail.
@@ -56,14 +56,14 @@ public final class Whitelist {
         Collections.singletonList(WhitelistLoader.loadFromResourceFiles(Whitelist.class, BASE_WHITELIST_FILES));
 
     /** The {@link ClassLoader} used to look up the whitelisted Java classes, constructors, methods, and fields. */
-    public final ClassLoader javaClassLoader;
+    public final ClassLoader classLoader;
 
     /** The {@link List} of all the whitelisted Painless classes. */
-    public final List<WhitelistClass> whitelistStructs;
+    public final List<WhitelistClass> whitelistClasses;
 
     /** Standard constructor.  All values must be not {@code null}. */
-    public Whitelist(ClassLoader javaClassLoader, List<WhitelistClass> whitelistStructs) {
-        this.javaClassLoader = Objects.requireNonNull(javaClassLoader);
-        this.whitelistStructs = Collections.unmodifiableList(Objects.requireNonNull(whitelistStructs));
+    public Whitelist(ClassLoader classLoader, List<WhitelistClass> whitelistClasses) {
+        this.classLoader = Objects.requireNonNull(classLoader);
+        this.whitelistClasses = Collections.unmodifiableList(Objects.requireNonNull(whitelistClasses));
     }
 }
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java
index 12aa5f5bdd634..0b216ae5c2953 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java
@@ -30,7 +30,7 @@
  * specific context, as long as multiple classes representing the same Java class have the same
  * class name and have legal constructor/method overloading they can be merged together.
  *
- * Classes in Painless allow for arity overloading for constructors and methods.  Arity overloading
+ * Classes in Painless allow for arity overloading for constructors and methods. Arity overloading
  * means that multiple constructors are allowed for a single class as long as they have a different
  * number of parameters, and multiples methods with the same name are allowed for a single class
  * as long as they have the same return type and a different number of parameters.
@@ -40,7 +40,7 @@
  */
 public final class WhitelistClass {
 
-    /** Information about where this class was white-listed from.  Can be used for error messages. */
+    /** Information about where this class was white-listed from. */
     public final String origin;
 
     /** The Java class name this class represents. */
@@ -49,7 +49,7 @@ public final class WhitelistClass {
     /**
      * Allow the Java class name to only be specified as the fully-qualified name.
      */
-    public final boolean onlyFQNJavaClassName;
+    public final boolean noImport;
 
     /** The {@link List} of whitelisted ({@link WhitelistConstructor}s) available to this class. */
     public final List<WhitelistConstructor> whitelistConstructors;
@@ -61,13 +61,14 @@ public final class WhitelistClass {
     public final List<WhitelistField> whitelistFields;
 
     /** Standard constructor. All values must be not {@code null}. */
-    public WhitelistClass(String origin, String javaClassName, boolean onlyFQNJavaClassName,
+    public WhitelistClass(String origin, String javaClassName, boolean noImport,
                           List<WhitelistConstructor> whitelistConstructors,
                           List<WhitelistMethod> whitelistMethods,
                           List<WhitelistField> whitelistFields) {
+
         this.origin = Objects.requireNonNull(origin);
         this.javaClassName = Objects.requireNonNull(javaClassName);
-        this.onlyFQNJavaClassName = onlyFQNJavaClassName;
+        this.noImport = noImport;
 
         this.whitelistConstructors = Collections.unmodifiableList(Objects.requireNonNull(whitelistConstructors));
         this.whitelistMethods = Collections.unmodifiableList(Objects.requireNonNull(whitelistMethods));
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java
index 0e70552760208..032ef397def01 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java
@@ -25,24 +25,24 @@
 
 /**
  * Constructor represents the equivalent of a Java constructor available as a whitelisted class
- * constructor within Painless.  Constructors for Painless classes may be accessed exactly as
- * constructors for Java classes are using the 'new' keyword.  Painless classes may have multiple
+ * constructor within Painless. Constructors for Painless classes may be accessed exactly as
+ * constructors for Java classes are using the 'new' keyword. Painless classes may have multiple
  * constructors as long as they comply with arity overloading described for {@link WhitelistClass}.
  */
 public final class WhitelistConstructor {
 
-    /** Information about where this constructor was whitelisted from.  Can be used for error messages. */
+    /** Information about where this constructor was whitelisted from. */
     public final String origin;
 
     /**
      * A {@link List} of {@link String}s that are the Painless type names for the parameters of the
      * constructor which can be used to look up the Java constructor through reflection.
      */
-    public final List<String> painlessParameterTypeNames;
+    public final List<String> canonicalTypeNameParameters;
 
     /** Standard constructor. All values must be not {@code null}. */
-    public WhitelistConstructor(String origin, List<String> painlessParameterTypeNames) {
+    public WhitelistConstructor(String origin, List<String> canonicalTypeNameParameters) {
         this.origin = Objects.requireNonNull(origin);
-        this.painlessParameterTypeNames = Collections.unmodifiableList(Objects.requireNonNull(painlessParameterTypeNames));
+        this.canonicalTypeNameParameters = Collections.unmodifiableList(Objects.requireNonNull(canonicalTypeNameParameters));
     }
 }
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java
index 116aea98fcf89..44ed31a227e1c 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java
@@ -23,24 +23,24 @@
 
 /**
  * Field represents the equivalent of a Java field available as a whitelisted class field
- * within Painless.  Fields for Painless classes may be accessed exactly as fields for Java classes
+ * within Painless. Fields for Painless classes may be accessed exactly as fields for Java classes
  * are using the '.' operator on an existing class variable/field.
  */
 public class WhitelistField {
 
-    /** Information about where this method was whitelisted from.  Can be used for error messages. */
+    /** Information about where this method was whitelisted from. */
     public final String origin;
 
-    /** The Java field name used to look up the Java field through reflection. */
-    public final String javaFieldName;
+    /** The field name used to look up the field reflection object. */
+    public final String fieldName;
 
-    /** The Painless type name for the field which can be used to look up the Java field through reflection. */
-    public final String painlessFieldTypeName;
+    /** The canonical type name for the field which can be used to look up the Java field through reflection. */
+    public final String canonicalTypeNameParameter;
 
     /** Standard constructor.  All values must be not {@code null}. */
-    public WhitelistField(String origin, String javaFieldName, String painlessFieldTypeName) {
+    public WhitelistField(String origin, String fieldName, String canonicalTypeNameParameter) {
         this.origin = Objects.requireNonNull(origin);
-        this.javaFieldName = Objects.requireNonNull(javaFieldName);
-        this.painlessFieldTypeName = Objects.requireNonNull(painlessFieldTypeName);
+        this.fieldName = Objects.requireNonNull(fieldName);
+        this.canonicalTypeNameParameter = Objects.requireNonNull(canonicalTypeNameParameter);
     }
 }
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java
index b104d03f1ea82..c59023b862280 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java
@@ -35,14 +35,14 @@
 public final class WhitelistLoader {
 
     /**
-     * Loads and creates a {@link Whitelist} from one to many text files.  The file paths are passed in as an array of
+     * Loads and creates a {@link Whitelist} from one to many text files. The file paths are passed in as an array of
      * {@link String}s with a single {@link Class} to be be used to load the resources where each {@link String}
-     * is the path of a single text file.  The {@link Class}'s {@link ClassLoader} will be used to lookup the Java
+     * is the path of a single text file. The {@link Class}'s {@link ClassLoader} will be used to lookup the Java
      * reflection objects for each individual {@link Class}, {@link Constructor}, {@link Method}, and {@link Field}
      * specified as part of the whitelist in the text file.
      *
      * A single pass is made through each file to collect all the information about each class, constructor, method,
-     * and field.  Most validation will be done at a later point after all whitelists have been gathered and their
+     * and field. Most validation will be done at a later point after all whitelists have been gathered and their
      * merging takes place.
      *
      * A painless type name is one of the following:
@@ -52,7 +52,7 @@ public final class WhitelistLoader {
      *     <li> fully-qualified Java type name - Any whitelisted Java class will have the equivalent name as
      *     a Painless type name with the exception that any dollar symbols used as part of inner classes will
      *     be replaced with dot symbols. </li>
-     *     <li> short Java type name - The text after the final dot symbol of any specified Java class.  A
+     *     <li> short Java type name - The text after the final dot symbol of any specified Java class. A
      *     short type Java name may be excluded by using the 'only_fqn' token during Painless class parsing
      *     as described later. </li>
      * </ul>
@@ -60,7 +60,7 @@ public final class WhitelistLoader {
      * The following can be parsed from each whitelist text file:
      * <ul>
      *   <li> Blank lines will be ignored by the parser. </li>
-     *   <li> Comments may be created starting with a pound '#' symbol and end with a newline.  These will
+     *   <li> Comments may be created starting with a pound '#' symbol and end with a newline. These will
      *   be ignored by the parser. </li>
      *   <li> Primitive types may be specified starting with 'class' and followed by the Java type name,
      *   an opening bracket, a newline, a closing bracket, and a final newline. </li>
@@ -93,10 +93,10 @@ public final class WhitelistLoader {
      *
      * Note there must be a one-to-one correspondence of Painless type names to Java type/class names.
      * If the same Painless type is defined across multiple files and the Java class is the same, all
-     * specified constructors, methods, and fields will be merged into a single Painless type.  The
+     * specified constructors, methods, and fields will be merged into a single Painless type. The
      * Painless dynamic type, 'def', used as part of constructor, method, and field definitions will
-     * be appropriately parsed and handled.  Painless complex types must be specified with the
-     * fully-qualified Java class name.  Method argument types, method return types, and field types
+     * be appropriately parsed and handled. Painless complex types must be specified with the
+     * fully-qualified Java class name. Method argument types, method return types, and field types
      * must be specified with Painless type names (def, fully-qualified, or short) as described earlier.
      *
      * The following example is used to create a single whitelist text file:
@@ -132,7 +132,7 @@ public final class WhitelistLoader {
      * }
      */
     public static Whitelist loadFromResourceFiles(Class<?> resource, String... filepaths) {
-        List<WhitelistClass> whitelistStructs = new ArrayList<>();
+        List<WhitelistClass> whitelistClasses = new ArrayList<>();
 
         // Execute a single pass through the whitelist text files.  This will gather all the
         // constructors, methods, augmented methods, and fields for each whitelisted class.
@@ -143,7 +143,7 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
             try (LineNumberReader reader = new LineNumberReader(
                 new InputStreamReader(resource.getResourceAsStream(filepath), StandardCharsets.UTF_8))) {
 
-                String whitelistStructOrigin = null;
+                String whitelistClassOrigin = null;
                 String javaClassName = null;
                 boolean onlyFQNJavaClassName = false;
                 List<WhitelistConstructor> whitelistConstructors = null;
@@ -178,7 +178,7 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
                             throw new IllegalArgumentException("invalid class definition: failed to parse class name [" + line + "]");
                         }
 
-                        whitelistStructOrigin = "[" + filepath + "]:[" + number + "]";
+                        whitelistClassOrigin = "[" + filepath + "]:[" + number + "]";
                         javaClassName = tokens[0];
 
                         // Reset all the constructors, methods, and fields to support a new class.
@@ -194,11 +194,11 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
                             throw new IllegalArgumentException("invalid class definition: extraneous closing bracket");
                         }
 
-                        whitelistStructs.add(new WhitelistClass(whitelistStructOrigin, javaClassName, onlyFQNJavaClassName,
+                        whitelistClasses.add(new WhitelistClass(whitelistClassOrigin, javaClassName, onlyFQNJavaClassName,
                             whitelistConstructors, whitelistMethods, whitelistFields));
 
                         // Set all the variables to null to ensure a new class definition is found before other parsable values.
-                        whitelistStructOrigin = null;
+                        whitelistClassOrigin = null;
                         javaClassName = null;
                         onlyFQNJavaClassName = false;
                         whitelistConstructors = null;
@@ -300,7 +300,7 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
         }
         ClassLoader loader = AccessController.doPrivileged((PrivilegedAction<ClassLoader>)resource::getClassLoader);
 
-        return new Whitelist(loader, whitelistStructs);
+        return new Whitelist(loader, whitelistClasses);
     }
 
     private WhitelistLoader() {}
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java
index df86619055b08..5cd023a3591ad 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java
@@ -25,52 +25,53 @@
 
 /**
  * Method represents the equivalent of a Java method available as a whitelisted class method
- * within Painless.  Methods for Painless classes may be accessed exactly as methods for Java classes
- * are using the '.' operator on an existing class variable/field.  Painless classes may have multiple
- * methods with the same name as long as they comply with arity overloading described for {@link WhitelistMethod}.
+ * within Painless. Methods for Painless classes may be accessed exactly as methods for Java classes
+ * are using the '.' operator on an existing class variable/field. Painless classes may have multiple
+ * methods with the same name as long as they comply with arity overloading described in
+ * {@link WhitelistClass}.
  *
  * Classes may also have additional methods that are not part of the Java class the class represents -
- * these are known as augmented methods.  An augmented method can be added to a class as a part of any
+ * these are known as augmented methods. An augmented method can be added to a class as a part of any
  * Java class as long as the method is static and the first parameter of the method is the Java class
- * represented by the class.  Note that the augmented method's parent Java class does not need to be
+ * represented by the class. Note that the augmented method's parent Java class does not need to be
  * whitelisted.
  */
 public class WhitelistMethod {
 
-    /** Information about where this method was whitelisted from.  Can be used for error messages. */
+    /** Information about where this method was whitelisted from. */
     public final String origin;
 
     /**
-     * The Java class name for the owner of an augmented method.  If the method is not augmented
+     * The class name for the owner of an augmented method. If the method is not augmented
      * this should be {@code null}.
      */
-    public final String javaAugmentedClassName;
+    public final String augmentedCanonicalClassName;
 
-    /** The Java method name used to look up the Java method through reflection. */
-    public final String javaMethodName;
+    /** The method name used to look up the method reflection object. */
+    public final String methodName;
 
     /**
-     * The Painless type name for the return type of the method which can be used to look up the Java
-     * method through reflection.
+     * The canonical type name for the return type.
      */
-    public final String painlessReturnTypeName;
+    public final String returnCanonicalTypeName;
 
     /**
-     * A {@link List} of {@link String}s that are the Painless type names for the parameters of the
-     * method which can be used to look up the Java method through reflection.
+     * A {@link List} of {@link String}s that are the canonical type names for the parameters of the
+     * method used to look up the method reflection object.
      */
-    public final List<String> painlessParameterTypeNames;
+    public final List<String> canonicalTypeNameParameters;
 
     /**
-     * Standard constructor. All values must be not {@code null} with the exception of jAugmentedClass;
-     * jAugmentedClass will be {@code null} unless the method is augmented as described in the class documentation.
+     * Standard constructor. All values must be not {@code null} with the exception of
+     * augmentedCanonicalClassName; augmentedCanonicalClassName will be {@code null} unless the method
+     * is augmented as described in the class documentation.
      */
-    public WhitelistMethod(String origin, String javaAugmentedClassName, String javaMethodName,
-                           String painlessReturnTypeName, List<String> painlessParameterTypeNames) {
+    public WhitelistMethod(String origin, String augmentedCanonicalClassName, String methodName,
+                           String returnCanonicalTypeName, List<String> canonicalTypeNameParameters) {
         this.origin = Objects.requireNonNull(origin);
-        this.javaAugmentedClassName = javaAugmentedClassName;
-        this.javaMethodName = javaMethodName;
-        this.painlessReturnTypeName = Objects.requireNonNull(painlessReturnTypeName);
-        this.painlessParameterTypeNames = Collections.unmodifiableList(Objects.requireNonNull(painlessParameterTypeNames));
+        this.augmentedCanonicalClassName = augmentedCanonicalClassName;
+        this.methodName = methodName;
+        this.returnCanonicalTypeName = Objects.requireNonNull(returnCanonicalTypeName);
+        this.canonicalTypeNameParameters = Collections.unmodifiableList(Objects.requireNonNull(canonicalTypeNameParameters));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index fe53a3c11001c..588fe8ef5f7cf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -41,421 +41,421 @@ public static PainlessCast getLegalCast(Location location, Class<?> actual, Clas
 
         if (actual == def.class) {
             if (expected == boolean.class) {
-                return PainlessCast.unboxTo(def.class, Boolean.class, explicit, boolean.class);
+                return PainlessCast.unboxTargetType(def.class, Boolean.class, explicit, boolean.class);
             } else if (expected == byte.class) {
-                return PainlessCast.unboxTo(def.class, Byte.class, explicit, byte.class);
+                return PainlessCast.unboxTargetType(def.class, Byte.class, explicit, byte.class);
             } else if (expected == short.class) {
-                return PainlessCast.unboxTo(def.class, Short.class, explicit, short.class);
+                return PainlessCast.unboxTargetType(def.class, Short.class, explicit, short.class);
             } else if (expected == char.class) {
-                return PainlessCast.unboxTo(def.class, Character.class, explicit, char.class);
+                return PainlessCast.unboxTargetType(def.class, Character.class, explicit, char.class);
             } else if (expected == int.class) {
-                return PainlessCast.unboxTo(def.class, Integer.class, explicit, int.class);
+                return PainlessCast.unboxTargetType(def.class, Integer.class, explicit, int.class);
             } else if (expected == long.class) {
-                return PainlessCast.unboxTo(def.class, Long.class, explicit, long.class);
+                return PainlessCast.unboxTargetType(def.class, Long.class, explicit, long.class);
             } else if (expected == float.class) {
-                return PainlessCast.unboxTo(def.class, Float.class, explicit, float.class);
+                return PainlessCast.unboxTargetType(def.class, Float.class, explicit, float.class);
             } else if (expected == double.class) {
-                return PainlessCast.unboxTo(def.class, Double.class, explicit, double.class);
+                return PainlessCast.unboxTargetType(def.class, Double.class, explicit, double.class);
             }
         } else if (actual == Object.class) {
             if (expected == byte.class && explicit && internal) {
-                return PainlessCast.unboxTo(Object.class, Byte.class, true, byte.class);
+                return PainlessCast.unboxTargetType(Object.class, Byte.class, true, byte.class);
             } else if (expected == short.class && explicit && internal) {
-                return PainlessCast.unboxTo(Object.class, Short.class, true, short.class);
+                return PainlessCast.unboxTargetType(Object.class, Short.class, true, short.class);
             } else if (expected == char.class && explicit && internal) {
-                return PainlessCast.unboxTo(Object.class, Character.class, true, char.class);
+                return PainlessCast.unboxTargetType(Object.class, Character.class, true, char.class);
             } else if (expected == int.class && explicit && internal) {
-                return PainlessCast.unboxTo(Object.class, Integer.class, true, int.class);
+                return PainlessCast.unboxTargetType(Object.class, Integer.class, true, int.class);
             } else if (expected == long.class && explicit && internal) {
-                return PainlessCast.unboxTo(Object.class, Long.class, true, long.class);
+                return PainlessCast.unboxTargetType(Object.class, Long.class, true, long.class);
             } else if (expected == float.class && explicit && internal) {
-                return PainlessCast.unboxTo(Object.class, Float.class, true, float.class);
+                return PainlessCast.unboxTargetType(Object.class, Float.class, true, float.class);
             } else if (expected == double.class && explicit && internal) {
-                return PainlessCast.unboxTo(Object.class, Double.class, true, double.class);
+                return PainlessCast.unboxTargetType(Object.class, Double.class, true, double.class);
             }
         } else if (actual == Number.class) {
             if (expected == byte.class && explicit && internal) {
-                return PainlessCast.unboxTo(Number.class, Byte.class, true, byte.class);
+                return PainlessCast.unboxTargetType(Number.class, Byte.class, true, byte.class);
             } else if (expected == short.class && explicit && internal) {
-                return PainlessCast.unboxTo(Number.class, Short.class, true, short.class);
+                return PainlessCast.unboxTargetType(Number.class, Short.class, true, short.class);
             } else if (expected == char.class && explicit && internal) {
-                return PainlessCast.unboxTo(Number.class, Character.class, true, char.class);
+                return PainlessCast.unboxTargetType(Number.class, Character.class, true, char.class);
             } else if (expected == int.class && explicit && internal) {
-                return PainlessCast.unboxTo(Number.class, Integer.class, true, int.class);
+                return PainlessCast.unboxTargetType(Number.class, Integer.class, true, int.class);
             } else if (expected == long.class && explicit && internal) {
-                return PainlessCast.unboxTo(Number.class, Long.class, true, long.class);
+                return PainlessCast.unboxTargetType(Number.class, Long.class, true, long.class);
             } else if (expected == float.class && explicit && internal) {
-                return PainlessCast.unboxTo(Number.class, Float.class, true, float.class);
+                return PainlessCast.unboxTargetType(Number.class, Float.class, true, float.class);
             } else if (expected == double.class && explicit && internal) {
-                return PainlessCast.unboxTo(Number.class, Double.class, true, double.class);
+                return PainlessCast.unboxTargetType(Number.class, Double.class, true, double.class);
             }
         } else if (actual == String.class) {
             if (expected == char.class && explicit) {
-                return PainlessCast.standard(String.class, char.class, true);
+                return PainlessCast.originalTypetoTargetType(String.class, char.class, true);
             }
         } else if (actual == boolean.class) {
             if (expected == def.class) {
-                return PainlessCast.boxFrom(Boolean.class, def.class, explicit, boolean.class);
+                return PainlessCast.boxOriginalType(Boolean.class, def.class, explicit, boolean.class);
             } else if (expected == Object.class && internal) {
-                return PainlessCast.boxFrom(Boolean.class, Object.class, explicit, boolean.class);
+                return PainlessCast.boxOriginalType(Boolean.class, Object.class, explicit, boolean.class);
             } else if (expected == Boolean.class && internal) {
-                return PainlessCast.boxTo(boolean.class, boolean.class, explicit, boolean.class);
+                return PainlessCast.boxTargetType(boolean.class, boolean.class, explicit, boolean.class);
             }
         } else if (actual == byte.class) {
             if (expected == def.class) {
-                return PainlessCast.boxFrom(Byte.class, def.class, explicit, byte.class);
+                return PainlessCast.boxOriginalType(Byte.class, def.class, explicit, byte.class);
             } else if (expected == Object.class && internal) {
-                return PainlessCast.boxFrom(Byte.class, Object.class, explicit, byte.class);
+                return PainlessCast.boxOriginalType(Byte.class, Object.class, explicit, byte.class);
             } else if (expected == Number.class && internal) {
-                return PainlessCast.boxFrom(Byte.class, Number.class, explicit, byte.class);
+                return PainlessCast.boxOriginalType(Byte.class, Number.class, explicit, byte.class);
             } else if (expected == short.class) {
-                return PainlessCast.standard(byte.class, short.class, explicit);
+                return PainlessCast.originalTypetoTargetType(byte.class, short.class, explicit);
             } else if (expected == char.class && explicit) {
-                return PainlessCast.standard(byte.class, char.class, true);
+                return PainlessCast.originalTypetoTargetType(byte.class, char.class, true);
             } else if (expected == int.class) {
-                return PainlessCast.standard(byte.class, int.class, explicit);
+                return PainlessCast.originalTypetoTargetType(byte.class, int.class, explicit);
             } else if (expected == long.class) {
-                return PainlessCast.standard(byte.class, long.class, explicit);
+                return PainlessCast.originalTypetoTargetType(byte.class, long.class, explicit);
             } else if (expected == float.class) {
-                return PainlessCast.standard(byte.class, float.class, explicit);
+                return PainlessCast.originalTypetoTargetType(byte.class, float.class, explicit);
             } else if (expected == double.class) {
-                return PainlessCast.standard(byte.class, double.class, explicit);
+                return PainlessCast.originalTypetoTargetType(byte.class, double.class, explicit);
             } else if (expected == Byte.class && internal) {
-                return PainlessCast.boxTo(byte.class, byte.class, explicit, byte.class);
+                return PainlessCast.boxTargetType(byte.class, byte.class, explicit, byte.class);
             } else if (expected == Short.class && internal) {
-                return PainlessCast.boxTo(byte.class, short.class, explicit, short.class);
+                return PainlessCast.boxTargetType(byte.class, short.class, explicit, short.class);
             } else if (expected == Character.class && explicit && internal) {
-                return PainlessCast.boxTo(byte.class, char.class, true, char.class);
+                return PainlessCast.boxTargetType(byte.class, char.class, true, char.class);
             } else if (expected == Integer.class && internal) {
-                return PainlessCast.boxTo(byte.class, int.class, explicit, int.class);
+                return PainlessCast.boxTargetType(byte.class, int.class, explicit, int.class);
             } else if (expected == Long.class && internal) {
-                return PainlessCast.boxTo(byte.class, long.class, explicit, long.class);
+                return PainlessCast.boxTargetType(byte.class, long.class, explicit, long.class);
             } else if (expected == Float.class && internal) {
-                return PainlessCast.boxTo(byte.class, float.class, explicit, float.class);
+                return PainlessCast.boxTargetType(byte.class, float.class, explicit, float.class);
             } else if (expected == Double.class && internal) {
-                return PainlessCast.boxTo(byte.class, double.class, explicit, double.class);
+                return PainlessCast.boxTargetType(byte.class, double.class, explicit, double.class);
             }
         } else if (actual == short.class) {
             if (expected == def.class) {
-                return PainlessCast.boxFrom(Short.class, def.class, explicit, short.class);
+                return PainlessCast.boxOriginalType(Short.class, def.class, explicit, short.class);
             } else if (expected == Object.class && internal) {
-                return PainlessCast.boxFrom(Short.class, Object.class, explicit, short.class);
+                return PainlessCast.boxOriginalType(Short.class, Object.class, explicit, short.class);
             } else if (expected == Number.class && internal) {
-                return PainlessCast.boxFrom(Short.class, Number.class, explicit, short.class);
+                return PainlessCast.boxOriginalType(Short.class, Number.class, explicit, short.class);
             } else if (expected == byte.class && explicit) {
-                return PainlessCast.standard(short.class, byte.class, true);
+                return PainlessCast.originalTypetoTargetType(short.class, byte.class, true);
             } else if (expected == char.class && explicit) {
-                return PainlessCast.standard(short.class, char.class, true);
+                return PainlessCast.originalTypetoTargetType(short.class, char.class, true);
             } else if (expected == int.class) {
-                return PainlessCast.standard(short.class, int.class, explicit);
+                return PainlessCast.originalTypetoTargetType(short.class, int.class, explicit);
             } else if (expected == long.class) {
-                return PainlessCast.standard(short.class, long.class, explicit);
+                return PainlessCast.originalTypetoTargetType(short.class, long.class, explicit);
             } else if (expected == float.class) {
-                return PainlessCast.standard(short.class, float.class, explicit);
+                return PainlessCast.originalTypetoTargetType(short.class, float.class, explicit);
             } else if (expected == double.class) {
-                return PainlessCast.standard(short.class, double.class, explicit);
+                return PainlessCast.originalTypetoTargetType(short.class, double.class, explicit);
             } else if (expected == Byte.class && explicit && internal) {
-                return PainlessCast.boxTo(short.class, byte.class, true, byte.class);
+                return PainlessCast.boxTargetType(short.class, byte.class, true, byte.class);
             } else if (expected == Short.class && internal) {
-                return PainlessCast.boxTo(short.class, short.class, explicit, short.class);
+                return PainlessCast.boxTargetType(short.class, short.class, explicit, short.class);
             } else if (expected == Character.class && explicit && internal) {
-                return PainlessCast.boxTo(short.class, char.class, true, char.class);
+                return PainlessCast.boxTargetType(short.class, char.class, true, char.class);
             } else if (expected == Integer.class && internal) {
-                return PainlessCast.boxTo(short.class, int.class, explicit, int.class);
+                return PainlessCast.boxTargetType(short.class, int.class, explicit, int.class);
             } else if (expected == Long.class && internal) {
-                return PainlessCast.boxTo(short.class, long.class, explicit, long.class);
+                return PainlessCast.boxTargetType(short.class, long.class, explicit, long.class);
             } else if (expected == Float.class && internal) {
-                return PainlessCast.boxTo(short.class, float.class, explicit, float.class);
+                return PainlessCast.boxTargetType(short.class, float.class, explicit, float.class);
             } else if (expected == Double.class && internal) {
-                return PainlessCast.boxTo(short.class, double.class, explicit, double.class);
+                return PainlessCast.boxTargetType(short.class, double.class, explicit, double.class);
             }
         } else if (actual == char.class) {
             if (expected == def.class) {
-                return PainlessCast.boxFrom(Character.class, def.class, explicit, char.class);
+                return PainlessCast.boxOriginalType(Character.class, def.class, explicit, char.class);
             } else if (expected == Object.class && internal) {
-                return PainlessCast.boxFrom(Character.class, Object.class, explicit, char.class);
+                return PainlessCast.boxOriginalType(Character.class, Object.class, explicit, char.class);
             } else if (expected == Number.class && internal) {
-                return PainlessCast.boxFrom(Character.class, Number.class, explicit, char.class);
+                return PainlessCast.boxOriginalType(Character.class, Number.class, explicit, char.class);
             } else if (expected == String.class) {
-                return PainlessCast.standard(char.class, String.class, explicit);
+                return PainlessCast.originalTypetoTargetType(char.class, String.class, explicit);
             } else if (expected == byte.class && explicit) {
-                return PainlessCast.standard(char.class, byte.class, true);
+                return PainlessCast.originalTypetoTargetType(char.class, byte.class, true);
             } else if (expected == short.class && explicit) {
-                return PainlessCast.standard(char.class, short.class, true);
+                return PainlessCast.originalTypetoTargetType(char.class, short.class, true);
             } else if (expected == int.class) {
-                return PainlessCast.standard(char.class, int.class, explicit);
+                return PainlessCast.originalTypetoTargetType(char.class, int.class, explicit);
             } else if (expected == long.class) {
-                return PainlessCast.standard(char.class, long.class, explicit);
+                return PainlessCast.originalTypetoTargetType(char.class, long.class, explicit);
             } else if (expected == float.class) {
-                return PainlessCast.standard(char.class, float.class, explicit);
+                return PainlessCast.originalTypetoTargetType(char.class, float.class, explicit);
             } else if (expected == double.class) {
-                return PainlessCast.standard(char.class, double.class, explicit);
+                return PainlessCast.originalTypetoTargetType(char.class, double.class, explicit);
             } else if (expected == Byte.class && explicit && internal) {
-                return PainlessCast.boxTo(char.class, byte.class, true, byte.class);
+                return PainlessCast.boxTargetType(char.class, byte.class, true, byte.class);
             } else if (expected == Short.class && internal) {
-                return PainlessCast.boxTo(char.class, short.class, explicit, short.class);
+                return PainlessCast.boxTargetType(char.class, short.class, explicit, short.class);
             } else if (expected == Character.class && internal) {
-                return PainlessCast.boxTo(char.class, char.class, true, char.class);
+                return PainlessCast.boxTargetType(char.class, char.class, true, char.class);
             } else if (expected == Integer.class && internal) {
-                return PainlessCast.boxTo(char.class, int.class, explicit, int.class);
+                return PainlessCast.boxTargetType(char.class, int.class, explicit, int.class);
             } else if (expected == Long.class && internal) {
-                return PainlessCast.boxTo(char.class, long.class, explicit, long.class);
+                return PainlessCast.boxTargetType(char.class, long.class, explicit, long.class);
             } else if (expected == Float.class && internal) {
-                return PainlessCast.boxTo(char.class, float.class, explicit, float.class);
+                return PainlessCast.boxTargetType(char.class, float.class, explicit, float.class);
             } else if (expected == Double.class && internal) {
-                return PainlessCast.boxTo(char.class, double.class, explicit, double.class);
+                return PainlessCast.boxTargetType(char.class, double.class, explicit, double.class);
             }
         } else if (actual == int.class) {
             if (expected == def.class) {
-                return PainlessCast.boxFrom(Integer.class, def.class, explicit, int.class);
+                return PainlessCast.boxOriginalType(Integer.class, def.class, explicit, int.class);
             } else if (expected == Object.class && internal) {
-                return PainlessCast.boxFrom(Integer.class, Object.class, explicit, int.class);
+                return PainlessCast.boxOriginalType(Integer.class, Object.class, explicit, int.class);
             } else if (expected == Number.class && internal) {
-                return PainlessCast.boxFrom(Integer.class, Number.class, explicit, int.class);
+                return PainlessCast.boxOriginalType(Integer.class, Number.class, explicit, int.class);
             } else if (expected == byte.class && explicit) {
-                return PainlessCast.standard(int.class, byte.class, true);
+                return PainlessCast.originalTypetoTargetType(int.class, byte.class, true);
             } else if (expected == char.class && explicit) {
-                return PainlessCast.standard(int.class, char.class, true);
+                return PainlessCast.originalTypetoTargetType(int.class, char.class, true);
             } else if (expected == short.class && explicit) {
-                return PainlessCast.standard(int.class, short.class, true);
+                return PainlessCast.originalTypetoTargetType(int.class, short.class, true);
             } else if (expected == long.class) {
-                return PainlessCast.standard(int.class, long.class, explicit);
+                return PainlessCast.originalTypetoTargetType(int.class, long.class, explicit);
             } else if (expected == float.class) {
-                return PainlessCast.standard(int.class, float.class, explicit);
+                return PainlessCast.originalTypetoTargetType(int.class, float.class, explicit);
             } else if (expected == double.class) {
-                return PainlessCast.standard(int.class, double.class, explicit);
+                return PainlessCast.originalTypetoTargetType(int.class, double.class, explicit);
             } else if (expected == Byte.class && explicit && internal) {
-                return PainlessCast.boxTo(int.class, byte.class, true, byte.class);
+                return PainlessCast.boxTargetType(int.class, byte.class, true, byte.class);
             } else if (expected == Short.class && explicit && internal) {
-                return PainlessCast.boxTo(int.class, short.class, true, short.class);
+                return PainlessCast.boxTargetType(int.class, short.class, true, short.class);
             } else if (expected == Character.class && explicit && internal) {
-                return PainlessCast.boxTo(int.class, char.class, true, char.class);
+                return PainlessCast.boxTargetType(int.class, char.class, true, char.class);
             } else if (expected == Integer.class && internal) {
-                return PainlessCast.boxTo(int.class, int.class, explicit, int.class);
+                return PainlessCast.boxTargetType(int.class, int.class, explicit, int.class);
             } else if (expected == Long.class && internal) {
-                return PainlessCast.boxTo(int.class, long.class, explicit, long.class);
+                return PainlessCast.boxTargetType(int.class, long.class, explicit, long.class);
             } else if (expected == Float.class && internal) {
-                return PainlessCast.boxTo(int.class, float.class, explicit, float.class);
+                return PainlessCast.boxTargetType(int.class, float.class, explicit, float.class);
             } else if (expected == Double.class && internal) {
-                return PainlessCast.boxTo(int.class, double.class, explicit, double.class);
+                return PainlessCast.boxTargetType(int.class, double.class, explicit, double.class);
             }
         } else if (actual == long.class) {
             if (expected == def.class) {
-                return PainlessCast.boxFrom(Long.class, def.class, explicit, long.class);
+                return PainlessCast.boxOriginalType(Long.class, def.class, explicit, long.class);
             } else if (expected == Object.class && internal) {
-                return PainlessCast.boxFrom(Long.class, Object.class, explicit, long.class);
+                return PainlessCast.boxOriginalType(Long.class, Object.class, explicit, long.class);
             } else if (expected == Number.class && internal) {
-                return PainlessCast.boxFrom(Long.class, Number.class, explicit, long.class);
+                return PainlessCast.boxOriginalType(Long.class, Number.class, explicit, long.class);
             } else if (expected == byte.class && explicit) {
-                return PainlessCast.standard(long.class, byte.class, true);
+                return PainlessCast.originalTypetoTargetType(long.class, byte.class, true);
             } else if (expected == char.class && explicit) {
-                return PainlessCast.standard(long.class, char.class, true);
+                return PainlessCast.originalTypetoTargetType(long.class, char.class, true);
             } else if (expected == short.class && explicit) {
-                return PainlessCast.standard(long.class, short.class, true);
+                return PainlessCast.originalTypetoTargetType(long.class, short.class, true);
             } else if (expected == int.class && explicit) {
-                return PainlessCast.standard(long.class, int.class, true);
+                return PainlessCast.originalTypetoTargetType(long.class, int.class, true);
             } else if (expected == float.class) {
-                return PainlessCast.standard(long.class, float.class, explicit);
+                return PainlessCast.originalTypetoTargetType(long.class, float.class, explicit);
             } else if (expected == double.class) {
-                return PainlessCast.standard(long.class, double.class, explicit);
+                return PainlessCast.originalTypetoTargetType(long.class, double.class, explicit);
             } else if (expected == Byte.class && explicit && internal) {
-                return PainlessCast.boxTo(long.class, byte.class, true, byte.class);
+                return PainlessCast.boxTargetType(long.class, byte.class, true, byte.class);
             } else if (expected == Short.class && explicit && internal) {
-                return PainlessCast.boxTo(long.class, short.class, true, short.class);
+                return PainlessCast.boxTargetType(long.class, short.class, true, short.class);
             } else if (expected == Character.class && explicit && internal) {
-                return PainlessCast.boxTo(long.class, char.class, true, char.class);
+                return PainlessCast.boxTargetType(long.class, char.class, true, char.class);
             } else if (expected == Integer.class && explicit && internal) {
-                return PainlessCast.boxTo(long.class, int.class, true, int.class);
+                return PainlessCast.boxTargetType(long.class, int.class, true, int.class);
             } else if (expected == Long.class && internal) {
-                return PainlessCast.boxTo(long.class, long.class, explicit, long.class);
+                return PainlessCast.boxTargetType(long.class, long.class, explicit, long.class);
             } else if (expected == Float.class && internal) {
-                return PainlessCast.boxTo(long.class, float.class, explicit, float.class);
+                return PainlessCast.boxTargetType(long.class, float.class, explicit, float.class);
             } else if (expected == Double.class && internal) {
-                return PainlessCast.boxTo(long.class, double.class, explicit, double.class);
+                return PainlessCast.boxTargetType(long.class, double.class, explicit, double.class);
             }
         } else if (actual == float.class) {
             if (expected == def.class) {
-                return PainlessCast.boxFrom(Float.class, def.class, explicit, float.class);
+                return PainlessCast.boxOriginalType(Float.class, def.class, explicit, float.class);
             } else if (expected == Object.class && internal) {
-                return PainlessCast.boxFrom(Float.class, Object.class, explicit, float.class);
+                return PainlessCast.boxOriginalType(Float.class, Object.class, explicit, float.class);
             } else if (expected == Number.class && internal) {
-                return PainlessCast.boxFrom(Float.class, Number.class, explicit, float.class);
+                return PainlessCast.boxOriginalType(Float.class, Number.class, explicit, float.class);
             } else if (expected == byte.class && explicit) {
-                return PainlessCast.standard(float.class, byte.class, true);
+                return PainlessCast.originalTypetoTargetType(float.class, byte.class, true);
             } else if (expected == char.class && explicit) {
-                return PainlessCast.standard(float.class, char.class, true);
+                return PainlessCast.originalTypetoTargetType(float.class, char.class, true);
             } else if (expected == short.class && explicit) {
-                return PainlessCast.standard(float.class, short.class, true);
+                return PainlessCast.originalTypetoTargetType(float.class, short.class, true);
             } else if (expected == int.class && explicit) {
-                return PainlessCast.standard(float.class, int.class, true);
+                return PainlessCast.originalTypetoTargetType(float.class, int.class, true);
             } else if (expected == long.class && explicit) {
-                return PainlessCast.standard(float.class, long.class, true);
+                return PainlessCast.originalTypetoTargetType(float.class, long.class, true);
             } else if (expected == double.class) {
-                return PainlessCast.standard(float.class, double.class, explicit);
+                return PainlessCast.originalTypetoTargetType(float.class, double.class, explicit);
             } else if (expected == Byte.class && explicit && internal) {
-                return PainlessCast.boxTo(float.class, byte.class, true, byte.class);
+                return PainlessCast.boxTargetType(float.class, byte.class, true, byte.class);
             } else if (expected == Short.class && explicit && internal) {
-                return PainlessCast.boxTo(float.class, short.class, true, short.class);
+                return PainlessCast.boxTargetType(float.class, short.class, true, short.class);
             } else if (expected == Character.class && explicit && internal) {
-                return PainlessCast.boxTo(float.class, char.class, true, char.class);
+                return PainlessCast.boxTargetType(float.class, char.class, true, char.class);
             } else if (expected == Integer.class && explicit && internal) {
-                return PainlessCast.boxTo(float.class, int.class, true, int.class);
+                return PainlessCast.boxTargetType(float.class, int.class, true, int.class);
             } else if (expected == Long.class && explicit && internal) {
-                return PainlessCast.boxTo(float.class, long.class, true, long.class);
+                return PainlessCast.boxTargetType(float.class, long.class, true, long.class);
             } else if (expected == Float.class && internal) {
-                return PainlessCast.boxTo(float.class, float.class, explicit, float.class);
+                return PainlessCast.boxTargetType(float.class, float.class, explicit, float.class);
             } else if (expected == Double.class && internal) {
-                return PainlessCast.boxTo(float.class, double.class, explicit, double.class);
+                return PainlessCast.boxTargetType(float.class, double.class, explicit, double.class);
             }
         } else if (actual == double.class) {
             if (expected == def.class) {
-                return PainlessCast.boxFrom(Double.class, def.class, explicit, double.class);
+                return PainlessCast.boxOriginalType(Double.class, def.class, explicit, double.class);
             } else if (expected == Object.class && internal) {
-                return PainlessCast.boxFrom(Double.class, Object.class, explicit, double.class);
+                return PainlessCast.boxOriginalType(Double.class, Object.class, explicit, double.class);
             } else if (expected == Number.class && internal) {
-                return PainlessCast.boxFrom(Double.class, Number.class, explicit, double.class);
+                return PainlessCast.boxOriginalType(Double.class, Number.class, explicit, double.class);
             } else if (expected == byte.class && explicit) {
-                return PainlessCast.standard(double.class, byte.class, true);
+                return PainlessCast.originalTypetoTargetType(double.class, byte.class, true);
             } else if (expected == char.class && explicit) {
-                return PainlessCast.standard(double.class, char.class, true);
+                return PainlessCast.originalTypetoTargetType(double.class, char.class, true);
             } else if (expected == short.class && explicit) {
-                return PainlessCast.standard(double.class, short.class, true);
+                return PainlessCast.originalTypetoTargetType(double.class, short.class, true);
             } else if (expected == int.class && explicit) {
-                return PainlessCast.standard(double.class, int.class, true);
+                return PainlessCast.originalTypetoTargetType(double.class, int.class, true);
             } else if (expected == long.class && explicit) {
-                return PainlessCast.standard(double.class, long.class, true);
+                return PainlessCast.originalTypetoTargetType(double.class, long.class, true);
             } else if (expected == float.class && explicit) {
-                return PainlessCast.standard(double.class, float.class, true);
+                return PainlessCast.originalTypetoTargetType(double.class, float.class, true);
             } else if (expected == Byte.class && explicit && internal) {
-                return PainlessCast.boxTo(double.class, byte.class, true, byte.class);
+                return PainlessCast.boxTargetType(double.class, byte.class, true, byte.class);
             } else if (expected == Short.class && explicit && internal) {
-                return PainlessCast.boxTo(double.class, short.class, true, short.class);
+                return PainlessCast.boxTargetType(double.class, short.class, true, short.class);
             } else if (expected == Character.class && explicit && internal) {
-                return PainlessCast.boxTo(double.class, char.class, true, char.class);
+                return PainlessCast.boxTargetType(double.class, char.class, true, char.class);
             } else if (expected == Integer.class && explicit && internal) {
-                return PainlessCast.boxTo(double.class, int.class, true, int.class);
+                return PainlessCast.boxTargetType(double.class, int.class, true, int.class);
             } else if (expected == Long.class && explicit && internal) {
-                return PainlessCast.boxTo(double.class, long.class, true, long.class);
+                return PainlessCast.boxTargetType(double.class, long.class, true, long.class);
             } else if (expected == Float.class && explicit && internal) {
-                return PainlessCast.boxTo(double.class, float.class, true, float.class);
+                return PainlessCast.boxTargetType(double.class, float.class, true, float.class);
             } else if (expected == Double.class && internal) {
-                return PainlessCast.boxTo(double.class, double.class, explicit, double.class);
+                return PainlessCast.boxTargetType(double.class, double.class, explicit, double.class);
             }
         } else if (actual == Boolean.class) {
             if (expected == boolean.class && internal) {
-                return PainlessCast.unboxFrom(boolean.class, boolean.class, explicit, boolean.class);
+                return PainlessCast.unboxOriginalType(boolean.class, boolean.class, explicit, boolean.class);
             }
         } else if (actual == Byte.class) {
             if (expected == byte.class && internal) {
-                return PainlessCast.unboxFrom(byte.class, byte.class, explicit, byte.class);
+                return PainlessCast.unboxOriginalType(byte.class, byte.class, explicit, byte.class);
             } else if (expected == short.class && internal) {
-                return PainlessCast.unboxFrom(byte.class, short.class, explicit, byte.class);
+                return PainlessCast.unboxOriginalType(byte.class, short.class, explicit, byte.class);
             } else if (expected == char.class && explicit && internal) {
-                return PainlessCast.unboxFrom(byte.class, char.class, true, byte.class);
+                return PainlessCast.unboxOriginalType(byte.class, char.class, true, byte.class);
             } else if (expected == int.class && internal) {
-                return PainlessCast.unboxFrom(byte.class, int.class, explicit, byte.class);
+                return PainlessCast.unboxOriginalType(byte.class, int.class, explicit, byte.class);
             } else if (expected == long.class && internal) {
-                return PainlessCast.unboxFrom(byte.class, long.class, explicit, byte.class);
+                return PainlessCast.unboxOriginalType(byte.class, long.class, explicit, byte.class);
             } else if (expected == float.class && internal) {
-                return PainlessCast.unboxFrom(byte.class, float.class, explicit, byte.class);
+                return PainlessCast.unboxOriginalType(byte.class, float.class, explicit, byte.class);
             } else if (expected == double.class && internal) {
-                return PainlessCast.unboxFrom(byte.class, double.class, explicit, byte.class);
+                return PainlessCast.unboxOriginalType(byte.class, double.class, explicit, byte.class);
             }
         } else if (actual == Short.class) {
             if (expected == byte.class && explicit && internal) {
-                return PainlessCast.unboxFrom(short.class, byte.class, true, short.class);
+                return PainlessCast.unboxOriginalType(short.class, byte.class, true, short.class);
             } else if (expected == short.class && internal) {
-                return PainlessCast.unboxFrom(short.class, short.class, explicit, short.class);
+                return PainlessCast.unboxOriginalType(short.class, short.class, explicit, short.class);
             } else if (expected == char.class && explicit && internal) {
-                return PainlessCast.unboxFrom(short.class, char.class, true, short.class);
+                return PainlessCast.unboxOriginalType(short.class, char.class, true, short.class);
             } else if (expected == int.class && internal) {
-                return PainlessCast.unboxFrom(short.class, int.class, explicit, short.class);
+                return PainlessCast.unboxOriginalType(short.class, int.class, explicit, short.class);
             } else if (expected == long.class && internal) {
-                return PainlessCast.unboxFrom(short.class, long.class, explicit, short.class);
+                return PainlessCast.unboxOriginalType(short.class, long.class, explicit, short.class);
             } else if (expected == float.class && internal) {
-                return PainlessCast.unboxFrom(short.class, float.class, explicit, short.class);
+                return PainlessCast.unboxOriginalType(short.class, float.class, explicit, short.class);
             } else if (expected == double.class && internal) {
-                return PainlessCast.unboxFrom(short.class, double.class, explicit, short.class);
+                return PainlessCast.unboxOriginalType(short.class, double.class, explicit, short.class);
             }
         } else if (actual == Character.class) {
             if (expected == byte.class && explicit && internal) {
-                return PainlessCast.unboxFrom(char.class, byte.class, true, char.class);
+                return PainlessCast.unboxOriginalType(char.class, byte.class, true, char.class);
             } else if (expected == short.class && explicit && internal) {
-                return PainlessCast.unboxFrom(char.class, short.class, true, char.class);
+                return PainlessCast.unboxOriginalType(char.class, short.class, true, char.class);
             } else if (expected == char.class && internal) {
-                return PainlessCast.unboxFrom(char.class, char.class, explicit, char.class);
+                return PainlessCast.unboxOriginalType(char.class, char.class, explicit, char.class);
             } else if (expected == int.class && internal) {
-                return PainlessCast.unboxFrom(char.class, int.class, explicit, char.class);
+                return PainlessCast.unboxOriginalType(char.class, int.class, explicit, char.class);
             } else if (expected == long.class && internal) {
-                return PainlessCast.unboxFrom(char.class, long.class, explicit, char.class);
+                return PainlessCast.unboxOriginalType(char.class, long.class, explicit, char.class);
             } else if (expected == float.class && internal) {
-                return PainlessCast.unboxFrom(char.class, float.class, explicit, char.class);
+                return PainlessCast.unboxOriginalType(char.class, float.class, explicit, char.class);
             } else if (expected == double.class && internal) {
-                return PainlessCast.unboxFrom(char.class, double.class, explicit, char.class);
+                return PainlessCast.unboxOriginalType(char.class, double.class, explicit, char.class);
             }
         } else if (actual == Integer.class) {
             if (expected == byte.class && explicit && internal) {
-                return PainlessCast.unboxFrom(int.class, byte.class, true, int.class);
+                return PainlessCast.unboxOriginalType(int.class, byte.class, true, int.class);
             } else if (expected == short.class && explicit && internal) {
-                return PainlessCast.unboxFrom(int.class, short.class, true, int.class);
+                return PainlessCast.unboxOriginalType(int.class, short.class, true, int.class);
             } else if (expected == char.class && explicit && internal) {
-                return PainlessCast.unboxFrom(int.class, char.class, true, int.class);
+                return PainlessCast.unboxOriginalType(int.class, char.class, true, int.class);
             } else if (expected == int.class && internal) {
-                return PainlessCast.unboxFrom(int.class, int.class, explicit, int.class);
+                return PainlessCast.unboxOriginalType(int.class, int.class, explicit, int.class);
             } else if (expected == long.class && internal) {
-                return PainlessCast.unboxFrom(int.class, long.class, explicit, int.class);
+                return PainlessCast.unboxOriginalType(int.class, long.class, explicit, int.class);
             } else if (expected == float.class && internal) {
-                return PainlessCast.unboxFrom(int.class, float.class, explicit, int.class);
+                return PainlessCast.unboxOriginalType(int.class, float.class, explicit, int.class);
             } else if (expected == double.class && internal) {
-                return PainlessCast.unboxFrom(int.class, double.class, explicit, int.class);
+                return PainlessCast.unboxOriginalType(int.class, double.class, explicit, int.class);
             }
         } else if (actual == Long.class) {
             if (expected == byte.class && explicit && internal) {
-                return PainlessCast.unboxFrom(long.class, byte.class, true, long.class);
+                return PainlessCast.unboxOriginalType(long.class, byte.class, true, long.class);
             } else if (expected == short.class && explicit && internal) {
-                return PainlessCast.unboxFrom(long.class, short.class, true, long.class);
+                return PainlessCast.unboxOriginalType(long.class, short.class, true, long.class);
             } else if (expected == char.class && explicit && internal) {
-                return PainlessCast.unboxFrom(long.class, char.class, true, long.class);
+                return PainlessCast.unboxOriginalType(long.class, char.class, true, long.class);
             } else if (expected == int.class && explicit && internal) {
-                return PainlessCast.unboxFrom(long.class, int.class, true, long.class);
+                return PainlessCast.unboxOriginalType(long.class, int.class, true, long.class);
             } else if (expected == long.class && internal) {
-                return PainlessCast.unboxFrom(long.class, long.class, explicit, long.class);
+                return PainlessCast.unboxOriginalType(long.class, long.class, explicit, long.class);
             } else if (expected == float.class && internal) {
-                return PainlessCast.unboxFrom(long.class, float.class, explicit, long.class);
+                return PainlessCast.unboxOriginalType(long.class, float.class, explicit, long.class);
             } else if (expected == double.class && internal) {
-                return PainlessCast.unboxFrom(long.class, double.class, explicit, long.class);
+                return PainlessCast.unboxOriginalType(long.class, double.class, explicit, long.class);
             }
         } else if (actual == Float.class) {
             if (expected == byte.class && explicit && internal) {
-                return PainlessCast.unboxFrom(float.class, byte.class, true, float.class);
+                return PainlessCast.unboxOriginalType(float.class, byte.class, true, float.class);
             } else if (expected == short.class && explicit && internal) {
-                return PainlessCast.unboxFrom(float.class, short.class, true, float.class);
+                return PainlessCast.unboxOriginalType(float.class, short.class, true, float.class);
             } else if (expected == char.class && explicit && internal) {
-                return PainlessCast.unboxFrom(float.class, char.class, true, float.class);
+                return PainlessCast.unboxOriginalType(float.class, char.class, true, float.class);
             } else if (expected == int.class && explicit && internal) {
-                return PainlessCast.unboxFrom(float.class, int.class, true, float.class);
+                return PainlessCast.unboxOriginalType(float.class, int.class, true, float.class);
             } else if (expected == long.class && explicit && internal) {
-                return PainlessCast.unboxFrom(float.class, long.class, true, float.class);
+                return PainlessCast.unboxOriginalType(float.class, long.class, true, float.class);
             } else if (expected == float.class && internal) {
-                return PainlessCast.unboxFrom(float.class, float.class, explicit, float.class);
+                return PainlessCast.unboxOriginalType(float.class, float.class, explicit, float.class);
             } else if (expected == double.class && internal) {
-                return PainlessCast.unboxFrom(float.class, double.class, explicit, float.class);
+                return PainlessCast.unboxOriginalType(float.class, double.class, explicit, float.class);
             }
         } else if (actual == Double.class) {
             if (expected == byte.class && explicit && internal) {
-                return PainlessCast.unboxFrom(double.class, byte.class, true, double.class);
+                return PainlessCast.unboxOriginalType(double.class, byte.class, true, double.class);
             } else if (expected == short.class && explicit && internal) {
-                return PainlessCast.unboxFrom(double.class, short.class, true, double.class);
+                return PainlessCast.unboxOriginalType(double.class, short.class, true, double.class);
             } else if (expected == char.class && explicit && internal) {
-                return PainlessCast.unboxFrom(double.class, char.class, true, double.class);
+                return PainlessCast.unboxOriginalType(double.class, char.class, true, double.class);
             } else if (expected == int.class && explicit && internal) {
-                return PainlessCast.unboxFrom(double.class, int.class, true, double.class);
+                return PainlessCast.unboxOriginalType(double.class, int.class, true, double.class);
             } else if (expected == long.class && explicit && internal) {
-                return PainlessCast.unboxFrom(double.class, long.class, true, double.class);
+                return PainlessCast.unboxOriginalType(double.class, long.class, true, double.class);
             } else if (expected == float.class && explicit && internal) {
-                return PainlessCast.unboxFrom(double.class, float.class, true, double.class);
+                return PainlessCast.unboxOriginalType(double.class, float.class, true, double.class);
             } else if (expected == double.class && internal) {
-                return PainlessCast.unboxFrom(double.class, double.class, explicit, double.class);
+                return PainlessCast.unboxOriginalType(double.class, double.class, explicit, double.class);
             }
         }
 
@@ -463,7 +463,7 @@ public static PainlessCast getLegalCast(Location location, Class<?> actual, Clas
                 (actual != void.class && expected == def.class) ||
                 expected.isAssignableFrom(actual)    ||
                 (actual.isAssignableFrom(expected) && explicit)) {
-            return PainlessCast.standard(actual, expected, explicit);
+            return PainlessCast.originalTypetoTargetType(actual, expected, explicit);
         } else {
             throw location.createError(new ClassCastException("Cannot cast from " +
                     "[" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "] to " +
@@ -472,8 +472,8 @@ public static PainlessCast getLegalCast(Location location, Class<?> actual, Clas
     }
 
     public static Object constCast(Location location, Object constant, PainlessCast cast) {
-        Class<?> fsort = cast.from;
-        Class<?> tsort = cast.to;
+        Class<?> fsort = cast.originalType;
+        Class<?> tsort = cast.targetType;
 
         if (fsort == tsort) {
             return constant;
@@ -499,11 +499,11 @@ public static Object constCast(Location location, Object constant, PainlessCast
             else if (tsort == double.class) return number.doubleValue();
             else {
                 throw location.createError(new IllegalStateException("Cannot cast from " +
-                    "[" + cast.from.getCanonicalName() + "] to [" + cast.to.getCanonicalName() + "]."));
+                    "[" + cast.originalType.getCanonicalName() + "] to [" + cast.targetType.getCanonicalName() + "]."));
             }
         } else {
             throw location.createError(new IllegalStateException("Cannot cast from " +
-                "[" + cast.from.getCanonicalName() + "] to [" + cast.to.getCanonicalName() + "]."));
+                "[" + cast.originalType.getCanonicalName() + "] to [" + cast.targetType.getCanonicalName() + "]."));
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 03345fcfff35a..97dddbdfe52c5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.bootstrap.BootstrapInfo;
+import org.elasticsearch.painless.Locals.LocalMethod;
 import org.elasticsearch.painless.antlr.Walker;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.node.SSource;
@@ -32,6 +33,7 @@
 import java.security.CodeSource;
 import java.security.SecureClassLoader;
 import java.security.cert.Certificate;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.elasticsearch.painless.WriterConstants.CLASS_NAME;
@@ -69,17 +71,14 @@ final class Compiler {
     /**
      * A secure class loader used to define Painless scripts.
      */
-    static final class Loader extends SecureClassLoader {
+    final class Loader extends SecureClassLoader {
         private final AtomicInteger lambdaCounter = new AtomicInteger(0);
-        private final PainlessLookup painlessLookup;
 
         /**
          * @param parent The parent ClassLoader.
          */
-        Loader(ClassLoader parent, PainlessLookup painlessLookup) {
+        Loader(ClassLoader parent) {
             super(parent);
-
-            this.painlessLookup = painlessLookup;
         }
 
         /**
@@ -90,7 +89,16 @@ static final class Loader extends SecureClassLoader {
          */
         @Override
         public Class<?> findClass(String name) throws ClassNotFoundException {
-            Class<?> found = painlessLookup.getClassFromBinaryName(name);
+            if (scriptClass.getName().equals(name)) {
+                return scriptClass;
+            }
+            if (factoryClass != null && factoryClass.getName().equals(name)) {
+                return factoryClass;
+            }
+            if (statefulFactoryClass != null && statefulFactoryClass.getName().equals(name)) {
+                return statefulFactoryClass;
+            }
+            Class<?> found = painlessLookup.canonicalTypeNameToType(name.replace('$', '.'));
 
             return found != null ? found : super.findClass(name);
         }
@@ -139,13 +147,23 @@ int newLambdaIdentifier() {
      * {@link Compiler}'s specified {@link PainlessLookup}.
      */
     public Loader createLoader(ClassLoader parent) {
-        return new Loader(parent, painlessLookup);
+        return new Loader(parent);
     }
 
     /**
-     * The class/interface the script is guaranteed to derive/implement.
+     * The class/interface the script will implement.
+     */
+    private final Class<?> scriptClass;
+
+    /**
+     * The class/interface to create the {@code scriptClass} instance.
      */
-    private final Class<?> base;
+    private final Class<?> factoryClass;
+
+    /**
+     * An optional class/interface to create the {@code factoryClass} instance.
+     */
+    private final Class<?> statefulFactoryClass;
 
     /**
      * The whitelist the script will use.
@@ -154,11 +172,15 @@ public Loader createLoader(ClassLoader parent) {
 
     /**
      * Standard constructor.
-     * @param base The class/interface the script is guaranteed to derive/implement.
+     * @param scriptClass The class/interface the script will implement.
+     * @param factoryClass An optional class/interface to create the {@code scriptClass} instance.
+     * @param statefulFactoryClass An optional class/interface to create the {@code factoryClass} instance.
      * @param painlessLookup The whitelist the script will use.
      */
-    Compiler(Class<?> base, PainlessLookup painlessLookup) {
-        this.base = base;
+    Compiler(Class<?> scriptClass, Class<?> factoryClass, Class<?> statefulFactoryClass, PainlessLookup painlessLookup) {
+        this.scriptClass = scriptClass;
+        this.factoryClass = factoryClass;
+        this.statefulFactoryClass = statefulFactoryClass;
         this.painlessLookup = painlessLookup;
     }
 
@@ -177,10 +199,10 @@ Constructor<?> compile(Loader loader, MainMethodReserved reserved, String name,
                 " plugin if a script longer than this length is a requirement.");
         }
 
-        ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, base);
+        ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SSource root = Walker.buildPainlessTree(scriptClassInfo, reserved, name, source, settings, painlessLookup,
                 null);
-        root.analyze(painlessLookup);
+        Map<String, LocalMethod> localMethods = root.analyze(painlessLookup);
         root.write();
 
         try {
@@ -189,6 +211,7 @@ Constructor<?> compile(Loader loader, MainMethodReserved reserved, String name,
             clazz.getField("$SOURCE").set(null, source);
             clazz.getField("$STATEMENTS").set(null, root.getStatements());
             clazz.getField("$DEFINITION").set(null, painlessLookup);
+            clazz.getField("$LOCALS").set(null, localMethods);
 
             return clazz.getConstructors()[0];
         } catch (Exception exception) { // Catch everything to let the user know this is something caused internally.
@@ -209,7 +232,7 @@ byte[] compile(String name, String source, CompilerSettings settings, Printer de
                 " plugin if a script longer than this length is a requirement.");
         }
 
-        ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, base);
+        ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SSource root = Walker.buildPainlessTree(scriptClassInfo, new MainMethodReserved(), name, source, settings, painlessLookup,
                 debugStream);
         root.analyze(painlessLookup);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 8a90f53b4fdfa..1e17d6024d4d1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.painless;
 
-import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.Locals.LocalMethod;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
@@ -37,6 +37,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
+
 /**
  * Support for dynamic type (def).
  * <p>
@@ -166,52 +168,6 @@ static MethodHandle arrayLengthGetter(Class<?> arrayType) {
         }
     }
 
-    /**
-     * Looks up method entry for a dynamic method call.
-     * <p>
-     * A dynamic method call for variable {@code x} of type {@code def} looks like:
-     * {@code x.method(args...)}
-     * <p>
-     * This method traverses {@code recieverClass}'s class hierarchy (including interfaces)
-     * until it finds a matching whitelisted method. If one is not found, it throws an exception.
-     * Otherwise it returns the matching method.
-     * <p>
-     * @params painlessLookup the whitelist
-     * @param receiverClass Class of the object to invoke the method on.
-     * @param name Name of the method.
-     * @param arity arity of method
-     * @return matching method to invoke. never returns null.
-     * @throws IllegalArgumentException if no matching whitelisted method was found.
-     */
-    static PainlessMethod lookupMethodInternal(PainlessLookup painlessLookup, Class<?> receiverClass, String name, int arity) {
-        String key = PainlessLookupUtility.buildPainlessMethodKey(name, arity);
-        // check whitelist for matching method
-        for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
-            PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz);
-
-            if (struct != null) {
-                PainlessMethod method = struct.methods.get(key);
-                if (method != null) {
-                    return method;
-                }
-            }
-
-            for (Class<?> iface : clazz.getInterfaces()) {
-                struct = painlessLookup.getPainlessStructFromJavaClass(iface);
-
-                if (struct != null) {
-                    PainlessMethod method = struct.methods.get(key);
-                    if (method != null) {
-                        return method;
-                    }
-                }
-            }
-        }
-
-        throw new IllegalArgumentException("Unable to find dynamic method [" + name + "] with [" + arity + "] arguments " +
-                                           "for class [" + receiverClass.getCanonicalName() + "].");
-    }
-
     /**
      * Looks up handle for a dynamic method call, with lambda replacement
      * <p>
@@ -232,13 +188,22 @@ static PainlessMethod lookupMethodInternal(PainlessLookup painlessLookup, Class<
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      * @throws Throwable if a method reference cannot be converted to an functional interface
      */
-    static MethodHandle lookupMethod(PainlessLookup painlessLookup, MethodHandles.Lookup methodHandlesLookup, MethodType callSiteType,
-                                     Class<?> receiverClass, String name, Object args[]) throws Throwable {
+    static MethodHandle lookupMethod(PainlessLookup painlessLookup, Map<String, LocalMethod> localMethods,
+            MethodHandles.Lookup methodHandlesLookup, MethodType callSiteType, Class<?> receiverClass, String name, Object args[])
+            throws Throwable {
+
          String recipeString = (String) args[0];
          int numArguments = callSiteType.parameterCount();
          // simple case: no lambdas
          if (recipeString.isEmpty()) {
-             return lookupMethodInternal(painlessLookup, receiverClass, name, numArguments - 1).handle;
+             PainlessMethod painlessMethod =  painlessLookup.lookupRuntimePainlessMethod(receiverClass, name, numArguments - 1);
+
+             if (painlessMethod == null) {
+                 throw new IllegalArgumentException("dynamic method " +
+                         "[" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + (numArguments - 1) + "] not found");
+             }
+
+             return painlessMethod.methodHandle;
          }
 
          // convert recipe string to a bitset for convenience (the code below should be refactored...)
@@ -261,8 +226,14 @@ static MethodHandle lookupMethod(PainlessLookup painlessLookup, MethodHandles.Lo
 
          // lookup the method with the proper arity, then we know everything (e.g. interface types of parameters).
          // based on these we can finally link any remaining lambdas that were deferred.
-         PainlessMethod method = lookupMethodInternal(painlessLookup, receiverClass, name, arity);
-         MethodHandle handle = method.handle;
+         PainlessMethod method = painlessLookup.lookupRuntimePainlessMethod(receiverClass, name, arity);
+
+        if (method == null) {
+            throw new IllegalArgumentException(
+                    "dynamic method [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + arity + "] not found");
+        }
+
+         MethodHandle handle = method.methodHandle;
 
          int replaced = 0;
          upTo = 1;
@@ -276,27 +247,29 @@ static MethodHandle lookupMethod(PainlessLookup painlessLookup, MethodHandles.Lo
                  String type = signature.substring(1, separator);
                  String call = signature.substring(separator+1, separator2);
                  int numCaptures = Integer.parseInt(signature.substring(separator2+1));
-                 Class<?> captures[] = new Class<?>[numCaptures];
-                 for (int capture = 0; capture < captures.length; capture++) {
-                     captures[capture] = callSiteType.parameterType(i + 1 + capture);
-                 }
                  MethodHandle filter;
-                 Class<?> interfaceType = method.arguments.get(i - 1 - replaced);
+                 Class<?> interfaceType = method.typeParameters.get(i - 1 - replaced);
                  if (signature.charAt(0) == 'S') {
                      // the implementation is strongly typed, now that we know the interface type,
                      // we have everything.
                      filter = lookupReferenceInternal(painlessLookup,
+                                                      localMethods,
                                                       methodHandlesLookup,
                                                       interfaceType,
                                                       type,
                                                       call,
-                                                      captures);
+                                                      numCaptures);
                  } else if (signature.charAt(0) == 'D') {
                      // the interface type is now known, but we need to get the implementation.
                      // this is dynamically based on the receiver type (and cached separately, underneath
                      // this cache). It won't blow up since we never nest here (just references)
+                     Class<?> captures[] = new Class<?>[numCaptures];
+                     for (int capture = 0; capture < captures.length; capture++) {
+                         captures[capture] = callSiteType.parameterType(i + 1 + capture);
+                     }
                      MethodType nestedType = MethodType.methodType(interfaceType, captures);
                      CallSite nested = DefBootstrap.bootstrap(painlessLookup,
+                                                              localMethods,
                                                               methodHandlesLookup,
                                                               call,
                                                               nestedType,
@@ -324,69 +297,44 @@ static MethodHandle lookupMethod(PainlessLookup painlessLookup, MethodHandles.Lo
       * This is just like LambdaMetaFactory, only with a dynamic type. The interface type is known,
       * so we simply need to lookup the matching implementation method based on receiver type.
       */
-    static MethodHandle lookupReference(PainlessLookup painlessLookup, MethodHandles.Lookup methodHandlesLookup, String interfaceClass,
-                                        Class<?> receiverClass, String name) throws Throwable {
-         Class<?> interfaceType = painlessLookup.getJavaClassFromPainlessType(interfaceClass);
-         PainlessMethod interfaceMethod = painlessLookup.getPainlessStructFromJavaClass(interfaceType).functionalMethod;
-         if (interfaceMethod == null) {
-             throw new IllegalArgumentException("Class [" + interfaceClass + "] is not a functional interface");
-         }
-         int arity = interfaceMethod.arguments.size();
-         PainlessMethod implMethod = lookupMethodInternal(painlessLookup, receiverClass, name, arity);
-        return lookupReferenceInternal(painlessLookup, methodHandlesLookup, interfaceType,
-                PainlessLookupUtility.typeToCanonicalTypeName(implMethod.target), implMethod.name, receiverClass);
+    static MethodHandle lookupReference(PainlessLookup painlessLookup, Map<String, LocalMethod> localMethods,
+            MethodHandles.Lookup methodHandlesLookup, String interfaceClass, Class<?> receiverClass, String name) throws Throwable {
+        Class<?> interfaceType = painlessLookup.canonicalTypeNameToType(interfaceClass);
+        if (interfaceType == null) {
+            throw new IllegalArgumentException("type [" + interfaceClass + "] not found");
+        }
+        PainlessMethod interfaceMethod = painlessLookup.lookupFunctionalInterfacePainlessMethod(interfaceType);
+        if (interfaceMethod == null) {
+            throw new IllegalArgumentException("Class [" + interfaceClass + "] is not a functional interface");
+        }
+        int arity = interfaceMethod.typeParameters.size();
+        PainlessMethod implMethod = painlessLookup.lookupRuntimePainlessMethod(receiverClass, name, arity);
+        if (implMethod == null) {
+            throw new IllegalArgumentException(
+                    "dynamic method [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + arity + "] not found");
+        }
+
+        return lookupReferenceInternal(painlessLookup, localMethods, methodHandlesLookup,
+            interfaceType, PainlessLookupUtility.typeToCanonicalTypeName(implMethod.targetClass),
+            implMethod.javaMethod.getName(), 1);
      }
 
      /** Returns a method handle to an implementation of clazz, given method reference signature. */
-    private static MethodHandle lookupReferenceInternal(PainlessLookup painlessLookup, MethodHandles.Lookup methodHandlesLookup,
-                                                        Class<?> clazz, String type, String call, Class<?>... captures)
-            throws Throwable {
-         final FunctionRef ref;
-         if ("this".equals(type)) {
-             // user written method
-             PainlessMethod interfaceMethod = painlessLookup.getPainlessStructFromJavaClass(clazz).functionalMethod;
-             if (interfaceMethod == null) {
-                 throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                         "to [" + PainlessLookupUtility.typeToCanonicalTypeName(clazz) + "], not a functional interface");
-             }
-             int arity = interfaceMethod.arguments.size() + captures.length;
-             final MethodHandle handle;
-             try {
-                 MethodHandle accessor = methodHandlesLookup.findStaticGetter(methodHandlesLookup.lookupClass(),
-                                                                              getUserFunctionHandleFieldName(call, arity),
-                                                                              MethodHandle.class);
-                 handle = (MethodHandle)accessor.invokeExact();
-             } catch (NoSuchFieldException | IllegalAccessException e) {
-                 // is it a synthetic method? If we generated the method ourselves, be more helpful. It can only fail
-                 // because the arity does not match the expected interface type.
-                 if (call.contains("$")) {
-                     throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name +
-                                                        "] in [" + clazz + "]");
-                 }
-                 throw new IllegalArgumentException("Unknown call [" + call + "] with [" + arity + "] arguments.");
-             }
-             ref = new FunctionRef(clazz, interfaceMethod, call, handle.type(), captures.length);
-         } else {
-             // whitelist lookup
-             ref = new FunctionRef(painlessLookup, clazz, type, call, captures.length);
-         }
-         final CallSite callSite = LambdaBootstrap.lambdaBootstrap(
-             methodHandlesLookup,
-             ref.interfaceMethodName,
-             ref.factoryMethodType,
-             ref.interfaceMethodType,
-             ref.delegateClassName,
-             ref.delegateInvokeType,
-             ref.delegateMethodName,
-             ref.delegateMethodType,
-             ref.isDelegateInterface ? 1 : 0
-         );
-         return callSite.dynamicInvoker().asType(MethodType.methodType(clazz, captures));
-     }
-
-     /** gets the field name used to lookup up the MethodHandle for a function. */
-     public static String getUserFunctionHandleFieldName(String name, int arity) {
-         return "handle$" + name + "$" + arity;
+    private static MethodHandle lookupReferenceInternal(PainlessLookup painlessLookup, Map<String, LocalMethod> localMethods,
+            MethodHandles.Lookup methodHandlesLookup, Class<?> clazz, String type, String call, int captures) throws Throwable {
+        final FunctionRef ref = FunctionRef.create(painlessLookup, localMethods, null, clazz, type, call, captures);
+        final CallSite callSite = LambdaBootstrap.lambdaBootstrap(
+            methodHandlesLookup,
+            ref.interfaceMethodName,
+            ref.factoryMethodType,
+            ref.interfaceMethodType,
+            ref.delegateClassName,
+            ref.delegateInvokeType,
+            ref.delegateMethodName,
+            ref.delegateMethodType,
+            ref.isDelegateInterface ? 1 : 0
+        );
+        return callSite.dynamicInvoker().asType(MethodType.methodType(clazz, ref.factoryMethodType.parameterArray()));
      }
 
     /**
@@ -417,27 +365,12 @@ public static String getUserFunctionHandleFieldName(String name, int arity) {
      */
     static MethodHandle lookupGetter(PainlessLookup painlessLookup, Class<?> receiverClass, String name) {
         // first try whitelist
-        for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
-            PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz);
-
-            if (struct != null) {
-                MethodHandle handle = struct.getterMethodHandles.get(name);
-                if (handle != null) {
-                    return handle;
-                }
-            }
+        MethodHandle getter = painlessLookup.lookupRuntimeGetterMethodHandle(receiverClass, name);
 
-            for (final Class<?> iface : clazz.getInterfaces()) {
-                struct = painlessLookup.getPainlessStructFromJavaClass(iface);
-
-                if (struct != null) {
-                    MethodHandle handle = struct.getterMethodHandles.get(name);
-                    if (handle != null) {
-                        return handle;
-                    }
-                }
-            }
+        if (getter != null) {
+            return getter;
         }
+
         // special case: arrays, maps, and lists
         if (receiverClass.isArray() && "length".equals(name)) {
             // arrays expose .length as a read-only getter
@@ -454,12 +387,12 @@ static MethodHandle lookupGetter(PainlessLookup painlessLookup, Class<?> receive
                 int index = Integer.parseInt(name);
                 return MethodHandles.insertArguments(LIST_GET, 1, index);
             } catch (NumberFormatException exception) {
-                throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "].");
+                throw new IllegalArgumentException("Illegal list shortcut value [" + name + "].");
             }
         }
 
-        throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " +
-                                           "for class [" + receiverClass.getCanonicalName() + "].");
+        throw new IllegalArgumentException(
+                "dynamic getter [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "] not found");
     }
 
     /**
@@ -488,27 +421,12 @@ static MethodHandle lookupGetter(PainlessLookup painlessLookup, Class<?> receive
      */
     static MethodHandle lookupSetter(PainlessLookup painlessLookup, Class<?> receiverClass, String name) {
         // first try whitelist
-        for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
-            PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz);
-
-            if (struct != null) {
-                MethodHandle handle = struct.setterMethodHandles.get(name);
-                if (handle != null) {
-                    return handle;
-                }
-            }
-
-            for (final Class<?> iface : clazz.getInterfaces()) {
-                struct = painlessLookup.getPainlessStructFromJavaClass(iface);
+        MethodHandle setter = painlessLookup.lookupRuntimeSetterMethodHandle(receiverClass, name);
 
-                if (struct != null) {
-                    MethodHandle handle = struct.setterMethodHandles.get(name);
-                    if (handle != null) {
-                        return handle;
-                    }
-                }
-            }
+        if (setter != null) {
+            return setter;
         }
+
         // special case: maps, and lists
         if (Map.class.isAssignableFrom(receiverClass)) {
             // maps allow access like mymap.key
@@ -522,12 +440,12 @@ static MethodHandle lookupSetter(PainlessLookup painlessLookup, Class<?> receive
                 int index = Integer.parseInt(name);
                 return MethodHandles.insertArguments(LIST_SET, 1, index);
             } catch (final NumberFormatException exception) {
-                throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "].");
+                throw new IllegalArgumentException("Illegal list shortcut value [" + name + "].");
             }
         }
 
-        throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " +
-                                           "for class [" + receiverClass.getCanonicalName() + "].");
+        throw new IllegalArgumentException(
+                "dynamic getter [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "] not found");
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 2fadaf30964a6..2488b6f218a7f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.painless.Locals.LocalMethod;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 
 import java.lang.invoke.CallSite;
@@ -28,6 +29,7 @@
 import java.lang.invoke.MethodType;
 import java.lang.invoke.MutableCallSite;
 import java.lang.invoke.WrongMethodTypeException;
+import java.util.Map;
 
 /**
  * Painless invokedynamic bootstrap for the call site.
@@ -105,19 +107,21 @@ static final class PIC extends MutableCallSite {
         static final int MAX_DEPTH = 5;
 
         private final PainlessLookup painlessLookup;
+        private final Map<String, LocalMethod> localMethods;
         private final MethodHandles.Lookup methodHandlesLookup;
         private final String name;
         private final int flavor;
         private final Object[] args;
         int depth; // pkg-protected for testing
 
-        PIC(PainlessLookup painlessLookup, MethodHandles.Lookup methodHandlesLookup,
-                String name, MethodType type, int initialDepth, int flavor, Object[] args) {
+        PIC(PainlessLookup painlessLookup, Map<String, LocalMethod> localMethods,
+                MethodHandles.Lookup methodHandlesLookup, String name, MethodType type, int initialDepth, int flavor, Object[] args) {
             super(type);
             if (type.parameterType(0) != Object.class) {
                 throw new BootstrapMethodError("The receiver type (1st arg) of invokedynamic descriptor must be Object.");
             }
             this.painlessLookup = painlessLookup;
+            this.localMethods = localMethods;
             this.methodHandlesLookup = methodHandlesLookup;
             this.name = name;
             this.flavor = flavor;
@@ -145,7 +149,7 @@ static boolean checkClass(Class<?> clazz, Object receiver) {
         private MethodHandle lookup(int flavor, String name, Class<?> receiver) throws Throwable {
             switch(flavor) {
                 case METHOD_CALL:
-                    return Def.lookupMethod(painlessLookup, methodHandlesLookup, type(), receiver, name, args);
+                    return Def.lookupMethod(painlessLookup, localMethods, methodHandlesLookup, type(), receiver, name, args);
                 case LOAD:
                     return Def.lookupGetter(painlessLookup, receiver, name);
                 case STORE:
@@ -157,7 +161,7 @@ private MethodHandle lookup(int flavor, String name, Class<?> receiver) throws T
                 case ITERATOR:
                     return Def.lookupIterator(receiver);
                 case REFERENCE:
-                    return Def.lookupReference(painlessLookup, methodHandlesLookup, (String) args[0], receiver, name);
+                    return Def.lookupReference(painlessLookup, localMethods, methodHandlesLookup, (String) args[0], receiver, name);
                 case INDEX_NORMALIZE:
                     return Def.lookupIndexNormalize(receiver);
                 default: throw new AssertionError();
@@ -432,8 +436,9 @@ static boolean checkBoth(Class<?> left, Class<?> right, Object leftObject, Objec
      * <p>
      * see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
      */
-    public static CallSite bootstrap(PainlessLookup painlessLookup, MethodHandles.Lookup methodHandlesLookup, String name,
-                                     MethodType type, int initialDepth, int flavor, Object... args) {
+    @SuppressWarnings("unchecked")
+    public static CallSite bootstrap(PainlessLookup painlessLookup, Map<String, LocalMethod> localMethods,
+            MethodHandles.Lookup methodHandlesLookup, String name, MethodType type, int initialDepth, int flavor, Object... args) {
         // validate arguments
         switch(flavor) {
             // "function-call" like things get a polymorphic cache
@@ -452,7 +457,7 @@ public static CallSite bootstrap(PainlessLookup painlessLookup, MethodHandles.Lo
                 if (args.length != numLambdas + 1) {
                     throw new BootstrapMethodError("Illegal number of parameters: expected " + numLambdas + " references");
                 }
-                return new PIC(painlessLookup, methodHandlesLookup, name, type, initialDepth, flavor, args);
+                return new PIC(painlessLookup, localMethods, methodHandlesLookup, name, type, initialDepth, flavor, args);
             case LOAD:
             case STORE:
             case ARRAY_LOAD:
@@ -462,7 +467,7 @@ public static CallSite bootstrap(PainlessLookup painlessLookup, MethodHandles.Lo
                 if (args.length > 0) {
                     throw new BootstrapMethodError("Illegal static bootstrap parameters for flavor: " + flavor);
                 }
-                return new PIC(painlessLookup, methodHandlesLookup, name, type, initialDepth, flavor, args);
+                return new PIC(painlessLookup, localMethods, methodHandlesLookup, name, type, initialDepth, flavor, args);
             case REFERENCE:
                 if (args.length != 1) {
                     throw new BootstrapMethodError("Invalid number of parameters for reference call");
@@ -470,7 +475,7 @@ public static CallSite bootstrap(PainlessLookup painlessLookup, MethodHandles.Lo
                 if (args[0] instanceof String == false) {
                     throw new BootstrapMethodError("Illegal parameter for reference call: " + args[0]);
                 }
-                return new PIC(painlessLookup, methodHandlesLookup, name, type, initialDepth, flavor, args);
+                return new PIC(painlessLookup, localMethods, methodHandlesLookup, name, type, initialDepth, flavor, args);
 
             // operators get monomorphic cache, with a generic impl for a fallback
             case UNARY_OPERATOR:
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index aa72724b93029..2580d7da3e8e7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -19,14 +19,18 @@
 
 package org.elasticsearch.painless;
 
-import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.Locals.LocalMethod;
+import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.objectweb.asm.Type;
 
 import java.lang.invoke.MethodType;
 import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.painless.WriterConstants.CLASS_NAME;
 import static org.objectweb.asm.Opcodes.H_INVOKEINTERFACE;
@@ -35,171 +39,205 @@
 import static org.objectweb.asm.Opcodes.H_NEWINVOKESPECIAL;
 
 /**
- * Reference to a function or lambda.
- * <p>
- * Once you have created one of these, you have "everything you need" to call {@link LambdaBootstrap}
- * either statically from bytecode with invokedynamic, or at runtime from Java.
+ * Contains all the values necessary to write the instruction to initiate a
+ * {@link LambdaBootstrap} for either a function reference or a user-defined
+ * lambda function.
  */
 public class FunctionRef {
 
-    /** functional interface method name */
-    public final String interfaceMethodName;
-    /** factory (CallSite) method signature */
-    public final MethodType factoryMethodType;
-    /** functional interface method signature */
-    public final MethodType interfaceMethodType;
-    /** class of the delegate method to be called */
-    public final String delegateClassName;
-    /** the invocation type of the delegate method */
-    public final int delegateInvokeType;
-    /** the name of the delegate method */
-    public final String delegateMethodName;
-    /** delegate method signature */
-    public final MethodType delegateMethodType;
+    /**
+     * Creates a new FunctionRef which will resolve {@code type::call} from the whitelist.
+     * @param painlessLookup the whitelist against which this script is being compiled
+     * @param localMethods user-defined and synthetic methods generated directly on the script class
+     * @param location the character number within the script at compile-time
+     * @param targetClass functional interface type to implement.
+     * @param typeName the left hand side of a method reference expression
+     * @param methodName the right hand side of a method reference expression
+     * @param numberOfCaptures number of captured arguments
+     */
+    public static FunctionRef create(PainlessLookup painlessLookup, Map<String, LocalMethod> localMethods, Location location,
+            Class<?> targetClass, String typeName, String methodName, int numberOfCaptures) {
 
-    /** interface method */
-    public final PainlessMethod interfaceMethod;
-    /** delegate method */
-    public final PainlessMethod delegateMethod;
+        Objects.requireNonNull(painlessLookup);
+        Objects.requireNonNull(targetClass);
+        Objects.requireNonNull(typeName);
+        Objects.requireNonNull(methodName);
 
-    /** factory method type descriptor */
-    public final String factoryDescriptor;
-    /** functional interface method as type */
-    public final Type interfaceType;
-    /** delegate method type method as type */
-    public final Type delegateType;
+        String targetClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
+        PainlessMethod interfaceMethod;
 
-    /** whether a call is made on a delegate interface */
-    public final boolean isDelegateInterface;
+        try {
+            interfaceMethod = painlessLookup.lookupFunctionalInterfacePainlessMethod(targetClass);
 
-    /**
-     * Creates a new FunctionRef, which will resolve {@code type::call} from the whitelist.
-     * @param painlessLookup the whitelist against which this script is being compiled
-     * @param expected functional interface type to implement.
-     * @param type the left hand side of a method reference expression
-     * @param call the right hand side of a method reference expression
-     * @param numCaptures number of captured arguments
-     */
-    public FunctionRef(PainlessLookup painlessLookup, Class<?> expected, String type, String call, int numCaptures) {
-        this(expected, painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod,
-                lookup(painlessLookup, expected, type, call, numCaptures > 0), numCaptures);
-    }
+            if (interfaceMethod == null) {
+                throw new IllegalArgumentException("cannot convert function reference [" + typeName + "::" + methodName + "] " +
+                        "to a non-functional interface [" + targetClassName + "]");
+            }
 
-    /**
-     * Creates a new FunctionRef (already resolved)
-     * @param expected functional interface type to implement
-     * @param interfaceMethod functional interface method
-     * @param delegateMethod implementation method
-     * @param numCaptures number of captured arguments
-     */
-    public FunctionRef(Class<?> expected, PainlessMethod interfaceMethod, PainlessMethod delegateMethod, int numCaptures) {
-        MethodType delegateMethodType = delegateMethod.getMethodType();
-
-        interfaceMethodName = interfaceMethod.name;
-        factoryMethodType = MethodType.methodType(expected,
-                delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount()));
-        interfaceMethodType = interfaceMethod.getMethodType().dropParameterTypes(0, 1);
-
-        // the Painless$Script class can be inferred if owner is null
-        if (delegateMethod.target == null) {
-            delegateClassName = CLASS_NAME;
-            isDelegateInterface = false;
-        } else if (delegateMethod.augmentation != null) {
-            delegateClassName = delegateMethod.augmentation.getName();
-            isDelegateInterface = delegateMethod.augmentation.isInterface();
-        } else {
-            delegateClassName = delegateMethod.target.getName();
-            isDelegateInterface = delegateMethod.target.isInterface();
-        }
+            String interfaceMethodName = interfaceMethod.javaMethod.getName();
+            MethodType interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1);
+            String delegateClassName;
+            boolean isDelegateInterface;
+            int delegateInvokeType;
+            String delegateMethodName;
+            MethodType delegateMethodType;
 
-        if ("<init>".equals(delegateMethod.name)) {
-            delegateInvokeType = H_NEWINVOKESPECIAL;
-        } else if (Modifier.isStatic(delegateMethod.modifiers)) {
-            delegateInvokeType = H_INVOKESTATIC;
-        } else if (delegateMethod.target.isInterface()) {
-            delegateInvokeType = H_INVOKEINTERFACE;
-        } else {
-            delegateInvokeType = H_INVOKEVIRTUAL;
-        }
+            Class<?> delegateMethodReturnType;
+            List<Class<?>> delegateMethodParameters;
+            int interfaceTypeParametersSize = interfaceMethod.typeParameters.size();
 
-        delegateMethodName = delegateMethod.name;
-        this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
+            if ("this".equals(typeName)) {
+                Objects.requireNonNull(localMethods);
 
-        this.interfaceMethod = interfaceMethod;
-        this.delegateMethod = delegateMethod;
+                if (numberOfCaptures < 0) {
+                    throw new IllegalStateException("internal error");
+                }
 
-        factoryDescriptor = factoryMethodType.toMethodDescriptorString();
-        interfaceType = Type.getMethodType(interfaceMethodType.toMethodDescriptorString());
-        delegateType = Type.getMethodType(this.delegateMethodType.toMethodDescriptorString());
-    }
+                String localMethodKey = Locals.buildLocalMethodKey(methodName, numberOfCaptures + interfaceTypeParametersSize);
+                LocalMethod localMethod = localMethods.get(localMethodKey);
 
-    /**
-     * Creates a new FunctionRef (low level).
-     * It is for runtime use only.
-     */
-    public FunctionRef(Class<?> expected,
-                       PainlessMethod interfaceMethod, String delegateMethodName, MethodType delegateMethodType, int numCaptures) {
-        interfaceMethodName = interfaceMethod.name;
-        factoryMethodType = MethodType.methodType(expected,
-            delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount()));
-        interfaceMethodType = interfaceMethod.getMethodType().dropParameterTypes(0, 1);
-
-        delegateClassName = CLASS_NAME;
-        delegateInvokeType = H_INVOKESTATIC;
-        this.delegateMethodName = delegateMethodName;
-        this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
-        isDelegateInterface = false;
+                if (localMethod == null) {
+                    throw new IllegalArgumentException("function reference [this::" + localMethodKey + "] " +
+                            "matching [" + targetClassName + ", " + interfaceMethodName + "/" + interfaceTypeParametersSize + "] " +
+                            "not found" + (localMethodKey.contains("$") ? " due to an incorrect number of arguments" : "")
+                    );
+                }
 
-        this.interfaceMethod = null;
-        delegateMethod = null;
+                delegateClassName = CLASS_NAME;
+                isDelegateInterface = false;
+                delegateInvokeType = H_INVOKESTATIC;
+                delegateMethodName = localMethod.name;
+                delegateMethodType = localMethod.methodType;
+
+                delegateMethodReturnType = localMethod.returnType;
+                delegateMethodParameters = localMethod.typeParameters;
+            } else if ("new".equals(methodName)) {
+                if (numberOfCaptures != 0) {
+                    throw new IllegalStateException("internal error");
+                }
 
-        factoryDescriptor = null;
-        interfaceType = null;
-        delegateType = null;
-    }
+                PainlessConstructor painlessConstructor = painlessLookup.lookupPainlessConstructor(typeName, interfaceTypeParametersSize);
 
-    /**
-     * Looks up {@code type::call} from the whitelist, and returns a matching method.
-     */
-    private static PainlessMethod lookup(PainlessLookup painlessLookup, Class<?> expected,
-                                         String type, String call, boolean receiverCaptured) {
-        // check its really a functional interface
-        // for e.g. Comparable
-        PainlessMethod method = painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod;
-        if (method == null) {
-            throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                    "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface");
-        }
+                if (painlessConstructor == null) {
+                    throw new IllegalArgumentException("function reference [" + typeName + "::new/" + interfaceTypeParametersSize + "] " +
+                            "matching [" + targetClassName + ", " + interfaceMethodName + "/" + interfaceTypeParametersSize + "] " +
+                            "not found");
+                }
+
+                delegateClassName = painlessConstructor.javaConstructor.getDeclaringClass().getName();
+                isDelegateInterface = false;
+                delegateInvokeType = H_NEWINVOKESPECIAL;
+                delegateMethodName = PainlessLookupUtility.CONSTRUCTOR_NAME;
+                delegateMethodType = painlessConstructor.methodType;
 
-        // lookup requested method
-        PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(painlessLookup.getJavaClassFromPainlessType(type));
-        final PainlessMethod impl;
-        // ctor ref
-        if ("new".equals(call)) {
-            impl = struct.constructors.get(PainlessLookupUtility.buildPainlessMethodKey("<init>", method.arguments.size()));
-        } else {
-            // look for a static impl first
-            PainlessMethod staticImpl =
-                    struct.staticMethods.get(PainlessLookupUtility.buildPainlessMethodKey(call, method.arguments.size()));
-            if (staticImpl == null) {
-                // otherwise a virtual impl
-                final int arity;
-                if (receiverCaptured) {
-                    // receiver captured
-                    arity = method.arguments.size();
+                delegateMethodReturnType = painlessConstructor.javaConstructor.getDeclaringClass();
+                delegateMethodParameters = painlessConstructor.typeParameters;
+            } else {
+                if (numberOfCaptures != 0 && numberOfCaptures != 1) {
+                    throw new IllegalStateException("internal error");
+                }
+
+                boolean captured = numberOfCaptures == 1;
+                PainlessMethod painlessMethod =
+                        painlessLookup.lookupPainlessMethod(typeName, true, methodName, interfaceTypeParametersSize);
+
+                if (painlessMethod == null) {
+                    painlessMethod = painlessLookup.lookupPainlessMethod(typeName, false, methodName,
+                            captured ? interfaceTypeParametersSize : interfaceTypeParametersSize - 1);
+
+                    if (painlessMethod == null) {
+                        throw new IllegalArgumentException(
+                                "function reference " + "[" + typeName + "::" + methodName + "/" + interfaceTypeParametersSize + "] " +
+                                "matching [" + targetClassName + ", " + interfaceMethodName + "/" + interfaceTypeParametersSize + "] " +
+                                "not found");
+                    }
+                } else if (captured) {
+                    throw new IllegalStateException("internal error");
+                }
+
+                delegateClassName = painlessMethod.javaMethod.getDeclaringClass().getName();
+                isDelegateInterface = painlessMethod.javaMethod.getDeclaringClass().isInterface();
+
+                if (Modifier.isStatic(painlessMethod.javaMethod.getModifiers())) {
+                    delegateInvokeType = H_INVOKESTATIC;
+                } else if (isDelegateInterface) {
+                    delegateInvokeType = H_INVOKEINTERFACE;
                 } else {
-                    // receiver passed
-                    arity = method.arguments.size() - 1;
+                    delegateInvokeType = H_INVOKEVIRTUAL;
+                }
+
+                delegateMethodName = painlessMethod.javaMethod.getName();
+                delegateMethodType = painlessMethod.methodType;
+
+                delegateMethodReturnType = painlessMethod.returnType;
+
+                if (delegateMethodType.parameterList().size() > painlessMethod.typeParameters.size()) {
+                    delegateMethodParameters = new ArrayList<>(painlessMethod.typeParameters);
+                    delegateMethodParameters.add(0, delegateMethodType.parameterType(0));
+                } else {
+                    delegateMethodParameters = painlessMethod.typeParameters;
                 }
-                impl = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(call, arity));
-            } else {
-                impl = staticImpl;
             }
+
+            if (location != null) {
+                for (int typeParameter = 0; typeParameter < interfaceTypeParametersSize; ++typeParameter) {
+                    Class<?> from = interfaceMethod.typeParameters.get(typeParameter);
+                    Class<?> to = delegateMethodParameters.get(numberOfCaptures + typeParameter);
+                    AnalyzerCaster.getLegalCast(location, from, to, false, true);
+                }
+
+                if (interfaceMethod.returnType != void.class) {
+                    AnalyzerCaster.getLegalCast(location, delegateMethodReturnType, interfaceMethod.returnType, false, true);
+                }
+            }
+
+            MethodType factoryMethodType = MethodType.methodType(targetClass,
+                    delegateMethodType.dropParameterTypes(numberOfCaptures, delegateMethodType.parameterCount()));
+            delegateMethodType = delegateMethodType.dropParameterTypes(0, numberOfCaptures);
+
+            return new FunctionRef(interfaceMethodName, interfaceMethodType,
+                    delegateClassName, isDelegateInterface, delegateInvokeType, delegateMethodName, delegateMethodType,
+                    factoryMethodType
+            );
+        } catch (IllegalArgumentException iae) {
+            if (location != null) {
+                throw location.createError(iae);
+            }
+
+            throw iae;
         }
-        if (impl == null) {
-            throw new IllegalArgumentException("Unknown reference [" + type + "::" + call + "] matching " +
-                                               "[" + expected + "]");
-        }
-        return impl;
+    }
+
+    /** functional interface method name */
+    public final String interfaceMethodName;
+    /** functional interface method signature */
+    public final MethodType interfaceMethodType;
+    /** class of the delegate method to be called */
+    public final String delegateClassName;
+    /** whether a call is made on a delegate interface */
+    public final boolean isDelegateInterface;
+    /** the invocation type of the delegate method */
+    public final int delegateInvokeType;
+    /** the name of the delegate method */
+    public final String delegateMethodName;
+    /** delegate method signature */
+    public final MethodType delegateMethodType;
+    /** factory (CallSite) method signature */
+    public final MethodType factoryMethodType;
+
+    private FunctionRef(
+            String interfaceMethodName, MethodType interfaceMethodType,
+            String delegateClassName, boolean isDelegateInterface,
+            int delegateInvokeType, String delegateMethodName, MethodType delegateMethodType,
+            MethodType factoryMethodType) {
+
+        this.interfaceMethodName = interfaceMethodName;
+        this.interfaceMethodType = interfaceMethodType;
+        this.delegateClassName = delegateClassName;
+        this.isDelegateInterface = isDelegateInterface;
+        this.delegateInvokeType = delegateInvokeType;
+        this.delegateMethodName = delegateMethodName;
+        this.delegateMethodType = delegateMethodType;
+        this.factoryMethodType = factoryMethodType;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
index 3fc8554b271e2..d95dc4266889b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
@@ -36,7 +36,6 @@
 import java.security.PrivilegedAction;
 
 import static java.lang.invoke.MethodHandles.Lookup;
-import static org.elasticsearch.painless.Compiler.Loader;
 import static org.elasticsearch.painless.WriterConstants.CLASS_VERSION;
 import static org.elasticsearch.painless.WriterConstants.CTOR_METHOD_NAME;
 import static org.elasticsearch.painless.WriterConstants.DELEGATE_BOOTSTRAP_HANDLE;
@@ -207,7 +206,7 @@ public static CallSite lambdaBootstrap(
             MethodType delegateMethodType,
             int isDelegateInterface)
             throws LambdaConversionException {
-        Loader loader = (Loader)lookup.lookupClass().getClassLoader();
+        Compiler.Loader loader = (Compiler.Loader)lookup.lookupClass().getClassLoader();
         String lambdaClassName = Type.getInternalName(lookup.lookupClass()) + "$$Lambda" + loader.newLambdaIdentifier();
         Type lambdaClassType = Type.getObjectType(lambdaClassName);
         Type delegateClassType = Type.getObjectType(delegateClassName.replace('.', '/'));
@@ -457,11 +456,11 @@ private static void endLambdaClass(ClassWriter cw) {
     }
 
     /**
-     * Defines the {@link Class} for the lambda class using the same {@link Loader}
+     * Defines the {@link Class} for the lambda class using the same {@link Compiler.Loader}
      * that originally defined the class for the Painless script.
      */
     private static Class<?> createLambdaClass(
-            Loader loader,
+            Compiler.Loader loader,
             ClassWriter cw,
             Type lambdaClassType) {
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index 804f6aa2b689c..e07c016ddd0e3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -22,8 +22,8 @@
 import org.elasticsearch.painless.ScriptClassInfo.MethodArgument;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
 
+import java.lang.invoke.MethodType;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -32,12 +32,39 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToJavaType;
 
 /**
  * Tracks user defined methods and variables across compilation phases.
  */
 public final class Locals {
 
+    /**
+     * Constructs a local method key used to lookup local methods from a painless class.
+     */
+    public static String buildLocalMethodKey(String methodName, int methodArity) {
+        return methodName + "/" + methodArity;
+    }
+
+    /**
+     * Stores information about methods directly callable on the generated script class.
+     */
+    public static class LocalMethod {
+        public final String name;
+        public final Class<?> returnType;
+        public final List<Class<?>> typeParameters;
+        public final MethodType methodType;
+
+        public LocalMethod(String name, Class<?> returnType, List<Class<?>> typeParameters, MethodType methodType) {
+            this.name = name;
+            this.returnType = returnType;
+            this.typeParameters = typeParameters;
+            this.methodType = methodType;
+        }
+    }
+
     /** Reserved word: loop counter */
     public static final String LOOP   = "#loop";
     /** Reserved word: unused */
@@ -50,7 +77,10 @@ public final class Locals {
 
     /** Creates a new local variable scope (e.g. loop) inside the current scope */
     public static Locals newLocalScope(Locals currentScope) {
-        return new Locals(currentScope);
+        Locals locals = new Locals(currentScope);
+        locals.methods = currentScope.methods;
+
+        return locals;
     }
 
     /**
@@ -58,9 +88,13 @@ public static Locals newLocalScope(Locals currentScope) {
      * <p>
      * This is just like {@link #newFunctionScope}, except the captured parameters are made read-only.
      */
-    public static Locals newLambdaScope(Locals programScope, Class<?> returnType, List<Parameter> parameters,
+    public static Locals newLambdaScope(Locals programScope, String name, Class<?> returnType, List<Parameter> parameters,
                                         int captureCount, int maxLoopCounter) {
         Locals locals = new Locals(programScope, programScope.painlessLookup, returnType, KEYWORDS);
+        locals.methods = programScope.methods;
+        List<Class<?>> typeParameters = parameters.stream().map(parameter -> typeToJavaType(parameter.clazz)).collect(Collectors.toList());
+        locals.methods.put(buildLocalMethodKey(name, parameters.size()), new LocalMethod(name, returnType, typeParameters,
+                MethodType.methodType(typeToJavaType(returnType), typeParameters)));
         for (int i = 0; i < parameters.size(); i++) {
             Parameter parameter = parameters.get(i);
             // TODO: allow non-captures to be r/w:
@@ -80,6 +114,7 @@ public static Locals newLambdaScope(Locals programScope, Class<?> returnType, Li
     /** Creates a new function scope inside the current scope */
     public static Locals newFunctionScope(Locals programScope, Class<?> returnType, List<Parameter> parameters, int maxLoopCounter) {
         Locals locals = new Locals(programScope, programScope.painlessLookup, returnType, KEYWORDS);
+        locals.methods = programScope.methods;
         for (Parameter parameter : parameters) {
             locals.addVariable(parameter.location, parameter.clazz, parameter.name, false);
         }
@@ -94,6 +129,7 @@ public static Locals newFunctionScope(Locals programScope, Class<?> returnType,
     public static Locals newMainMethodScope(ScriptClassInfo scriptClassInfo, Locals programScope, int maxLoopCounter) {
         Locals locals = new Locals(
             programScope, programScope.painlessLookup, scriptClassInfo.getExecuteMethodReturnType(), KEYWORDS);
+        locals.methods = programScope.methods;
         // This reference. Internal use only.
         locals.defineVariable(null, Object.class, THIS, true);
 
@@ -110,9 +146,10 @@ public static Locals newMainMethodScope(ScriptClassInfo scriptClassInfo, Locals
     }
 
     /** Creates a new program scope: the list of methods. It is the parent for all methods */
-    public static Locals newProgramScope(PainlessLookup painlessLookup, Collection<PainlessMethod> methods) {
+    public static Locals newProgramScope(PainlessLookup painlessLookup, Collection<LocalMethod> methods) {
         Locals locals = new Locals(null, painlessLookup, null, null);
-        for (PainlessMethod method : methods) {
+        locals.methods = new HashMap<>();
+        for (LocalMethod method : methods) {
             locals.addMethod(method);
         }
         return locals;
@@ -143,15 +180,8 @@ public Variable getVariable(Location location, String name) {
     }
 
     /** Looks up a method. Returns null if the method does not exist. */
-    public PainlessMethod getMethod(String key) {
-        PainlessMethod method = lookupMethod(key);
-        if (method != null) {
-            return method;
-        }
-        if (parent != null) {
-            return parent.getMethod(key);
-        }
-        return null;
+    public LocalMethod getMethod(String methodName, int methodArity) {
+        return methods.get(buildLocalMethodKey(methodName, methodArity));
     }
 
     /** Creates a new variable. Throws IAE if the variable has already been defined (even in a parent) or reserved. */
@@ -199,7 +229,7 @@ public PainlessLookup getPainlessLookup() {
     // variable name -> variable
     private Map<String,Variable> variables;
     // method name+arity -> methods
-    private Map<String,PainlessMethod> methods;
+    private Map<String,LocalMethod> methods;
 
     /**
      * Create a new Locals
@@ -236,15 +266,10 @@ private Variable lookupVariable(Location location, String name) {
         return variables.get(name);
     }
 
-    /** Looks up a method at this scope only. Returns null if the method does not exist. */
-    private PainlessMethod lookupMethod(String key) {
-        if (methods == null) {
-            return null;
-        }
-        return methods.get(key);
+    public Map<String, LocalMethod> getMethods() {
+        return Collections.unmodifiableMap(methods);
     }
 
-
     /** Defines a variable at this scope internally. */
     private Variable defineVariable(Location location, Class<?> type, String name, boolean readonly) {
         if (variables == null) {
@@ -256,15 +281,10 @@ private Variable defineVariable(Location location, Class<?> type, String name, b
         return variable;
     }
 
-    private void addMethod(PainlessMethod method) {
-        if (methods == null) {
-            methods = new HashMap<>();
-        }
-        methods.put(PainlessLookupUtility.buildPainlessMethodKey(method.name, method.arguments.size()), method);
-        // TODO: check result
+    private void addMethod(LocalMethod method) {
+        methods.put(buildLocalMethodKey(method.name, method.typeParameters.size()), method);
     }
 
-
     private int getNextSlot() {
         return nextSlotNumber;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index c339e7bfb2613..dca638b3dddac 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.lookup.PainlessCast;
+import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.Label;
@@ -28,6 +29,7 @@
 import org.objectweb.asm.commons.GeneratorAdapter;
 import org.objectweb.asm.commons.Method;
 
+import java.lang.reflect.Modifier;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -54,6 +56,7 @@
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_IMPLICIT;
 import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE;
 import static org.elasticsearch.painless.WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
+import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
 import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
 import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_BOOLEAN;
@@ -132,52 +135,52 @@ public void writeLoopCounter(int slot, int count, Location location) {
 
     public void writeCast(PainlessCast cast) {
         if (cast != null) {
-            if (cast.from == char.class && cast.to == String.class) {
+            if (cast.originalType == char.class && cast.targetType == String.class) {
                 invokeStatic(UTILITY_TYPE, CHAR_TO_STRING);
-            } else if (cast.from == String.class && cast.to == char.class) {
+            } else if (cast.originalType == String.class && cast.targetType == char.class) {
                 invokeStatic(UTILITY_TYPE, STRING_TO_CHAR);
-            } else if (cast.unboxFrom != null) {
-                unbox(getType(cast.unboxFrom));
-                writeCast(cast.from, cast.to);
-            } else if (cast.unboxTo != null) {
-                if (cast.from == def.class) {
-                    if (cast.explicit) {
-                        if      (cast.to == Boolean.class)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_BOOLEAN);
-                        else if (cast.to == Byte.class)      invokeStatic(DEF_UTIL_TYPE, DEF_TO_BYTE_EXPLICIT);
-                        else if (cast.to == Short.class)     invokeStatic(DEF_UTIL_TYPE, DEF_TO_SHORT_EXPLICIT);
-                        else if (cast.to == Character.class) invokeStatic(DEF_UTIL_TYPE, DEF_TO_CHAR_EXPLICIT);
-                        else if (cast.to == Integer.class)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_INT_EXPLICIT);
-                        else if (cast.to == Long.class)      invokeStatic(DEF_UTIL_TYPE, DEF_TO_LONG_EXPLICIT);
-                        else if (cast.to == Float.class)     invokeStatic(DEF_UTIL_TYPE, DEF_TO_FLOAT_EXPLICIT);
-                        else if (cast.to == Double.class)    invokeStatic(DEF_UTIL_TYPE, DEF_TO_DOUBLE_EXPLICIT);
+            } else if (cast.unboxOriginalType != null) {
+                unbox(getType(cast.unboxOriginalType));
+                writeCast(cast.originalType, cast.targetType);
+            } else if (cast.unboxTargetType != null) {
+                if (cast.originalType == def.class) {
+                    if (cast.explicitCast) {
+                        if      (cast.targetType == Boolean.class)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_BOOLEAN);
+                        else if (cast.targetType == Byte.class)      invokeStatic(DEF_UTIL_TYPE, DEF_TO_BYTE_EXPLICIT);
+                        else if (cast.targetType == Short.class)     invokeStatic(DEF_UTIL_TYPE, DEF_TO_SHORT_EXPLICIT);
+                        else if (cast.targetType == Character.class) invokeStatic(DEF_UTIL_TYPE, DEF_TO_CHAR_EXPLICIT);
+                        else if (cast.targetType == Integer.class)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_INT_EXPLICIT);
+                        else if (cast.targetType == Long.class)      invokeStatic(DEF_UTIL_TYPE, DEF_TO_LONG_EXPLICIT);
+                        else if (cast.targetType == Float.class)     invokeStatic(DEF_UTIL_TYPE, DEF_TO_FLOAT_EXPLICIT);
+                        else if (cast.targetType == Double.class)    invokeStatic(DEF_UTIL_TYPE, DEF_TO_DOUBLE_EXPLICIT);
                         else {
                             throw new IllegalStateException("Illegal tree structure.");
                         }
                     } else {
-                        if      (cast.to == Boolean.class)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_BOOLEAN);
-                        else if (cast.to == Byte.class)      invokeStatic(DEF_UTIL_TYPE, DEF_TO_BYTE_IMPLICIT);
-                        else if (cast.to == Short.class)     invokeStatic(DEF_UTIL_TYPE, DEF_TO_SHORT_IMPLICIT);
-                        else if (cast.to == Character.class) invokeStatic(DEF_UTIL_TYPE, DEF_TO_CHAR_IMPLICIT);
-                        else if (cast.to == Integer.class)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_INT_IMPLICIT);
-                        else if (cast.to == Long.class)      invokeStatic(DEF_UTIL_TYPE, DEF_TO_LONG_IMPLICIT);
-                        else if (cast.to == Float.class)     invokeStatic(DEF_UTIL_TYPE, DEF_TO_FLOAT_IMPLICIT);
-                        else if (cast.to == Double.class)    invokeStatic(DEF_UTIL_TYPE, DEF_TO_DOUBLE_IMPLICIT);
+                        if      (cast.targetType == Boolean.class)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_BOOLEAN);
+                        else if (cast.targetType == Byte.class)      invokeStatic(DEF_UTIL_TYPE, DEF_TO_BYTE_IMPLICIT);
+                        else if (cast.targetType == Short.class)     invokeStatic(DEF_UTIL_TYPE, DEF_TO_SHORT_IMPLICIT);
+                        else if (cast.targetType == Character.class) invokeStatic(DEF_UTIL_TYPE, DEF_TO_CHAR_IMPLICIT);
+                        else if (cast.targetType == Integer.class)   invokeStatic(DEF_UTIL_TYPE, DEF_TO_INT_IMPLICIT);
+                        else if (cast.targetType == Long.class)      invokeStatic(DEF_UTIL_TYPE, DEF_TO_LONG_IMPLICIT);
+                        else if (cast.targetType == Float.class)     invokeStatic(DEF_UTIL_TYPE, DEF_TO_FLOAT_IMPLICIT);
+                        else if (cast.targetType == Double.class)    invokeStatic(DEF_UTIL_TYPE, DEF_TO_DOUBLE_IMPLICIT);
                         else {
                             throw new IllegalStateException("Illegal tree structure.");
                         }
                     }
                 } else {
-                    writeCast(cast.from, cast.to);
-                    unbox(getType(cast.unboxTo));
+                    writeCast(cast.originalType, cast.targetType);
+                    unbox(getType(cast.unboxTargetType));
                 }
-            } else if (cast.boxFrom != null) {
-                box(getType(cast.boxFrom));
-                writeCast(cast.from, cast.to);
-            } else if (cast.boxTo != null) {
-                writeCast(cast.from, cast.to);
-                box(getType(cast.boxTo));
+            } else if (cast.boxOriginalType != null) {
+                box(getType(cast.boxOriginalType));
+                writeCast(cast.originalType, cast.targetType);
+            } else if (cast.boxTargetType != null) {
+                writeCast(cast.originalType, cast.targetType);
+                box(getType(cast.boxTargetType));
             } else {
-                writeCast(cast.from, cast.to);
+                writeCast(cast.originalType, cast.targetType);
             }
         }
     }
@@ -415,4 +418,40 @@ public void invokeDefCall(String name, Type methodType, int flavor, Object... pa
         System.arraycopy(params, 0, args, 2, params.length);
         invokeDynamic(name, methodType.getDescriptor(), DEF_BOOTSTRAP_HANDLE, args);
     }
+
+    public void invokeMethodCall(PainlessMethod painlessMethod) {
+        Type type = Type.getType(painlessMethod.javaMethod.getDeclaringClass());
+        Method method = Method.getMethod(painlessMethod.javaMethod);
+
+        if (Modifier.isStatic(painlessMethod.javaMethod.getModifiers())) {
+            // invokeStatic assumes that the owner class is not an interface, so this is a
+            // special case for interfaces where the interface method boolean needs to be set to
+            // true to reference the appropriate class constant when calling a static interface
+            // method since java 8 did not check, but java 9 and 10 do
+            if (painlessMethod.javaMethod.getDeclaringClass().isInterface()) {
+                visitMethodInsn(Opcodes.INVOKESTATIC, type.getInternalName(),
+                        painlessMethod.javaMethod.getName(), painlessMethod.methodType.toMethodDescriptorString(), true);
+            } else {
+                invokeStatic(type, method);
+            }
+        } else if (painlessMethod.javaMethod.getDeclaringClass().isInterface()) {
+            invokeInterface(type, method);
+        } else {
+            invokeVirtual(type, method);
+        }
+    }
+
+    public void invokeLambdaCall(FunctionRef functionRef) {
+        invokeDynamic(
+                functionRef.interfaceMethodName,
+                functionRef.factoryMethodType.toMethodDescriptorString(),
+                LAMBDA_BOOTSTRAP_HANDLE,
+                Type.getMethodType(functionRef.interfaceMethodType.toMethodDescriptorString()),
+                functionRef.delegateClassName,
+                functionRef.delegateInvokeType,
+                functionRef.delegateMethodName,
+                Type.getMethodType(functionRef.delegateMethodType.toMethodDescriptorString()),
+                functionRef.isDelegateInterface ? 1 : 0
+        );
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java
index 7bef028c7d1ca..e4988103bc681 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java
@@ -57,7 +57,7 @@ public Map<String, List<String>> getHeaders(PainlessLookup painlessLookup) {
         if (objectToExplain != null) {
             toString = objectToExplain.toString();
             javaClassName = objectToExplain.getClass().getName();
-            PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(objectToExplain.getClass());
+            PainlessClass struct = painlessLookup.lookupPainlessClass(objectToExplain.getClass());
             if (struct != null) {
                 painlessClassName = PainlessLookupUtility.typeToCanonicalTypeName(objectToExplain.getClass());
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
index a5a9823d13018..3a2a6d1452df1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
@@ -102,10 +102,10 @@ public PainlessScriptEngine(Settings settings, Map<ScriptContext<?>, List<Whitel
         for (Map.Entry<ScriptContext<?>, List<Whitelist>> entry : contexts.entrySet()) {
             ScriptContext<?> context = entry.getKey();
             if (context.instanceClazz.equals(SearchScript.class) || context.instanceClazz.equals(ExecutableScript.class)) {
-                contextsToCompilers.put(context, new Compiler(GenericElasticsearchScript.class,
+                contextsToCompilers.put(context, new Compiler(GenericElasticsearchScript.class, null, null,
                         PainlessLookupBuilder.buildFromWhitelists(entry.getValue())));
             } else {
-                contextsToCompilers.put(context, new Compiler(context.instanceClazz,
+                contextsToCompilers.put(context, new Compiler(context.instanceClazz, context.factoryClazz, context.statefulFactoryClazz,
                         PainlessLookupBuilder.buildFromWhitelists(entry.getValue())));
             }
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
index 6d4b455269616..345db46f8875f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
@@ -190,7 +190,7 @@ private static Class<?> definitionTypeForClass(PainlessLookup painlessLookup, Cl
             componentType = componentType.getComponentType();
         }
 
-        if (painlessLookup.getPainlessStructFromJavaClass(componentType) == null) {
+        if (painlessLookup.lookupPainlessClass(componentType) == null) {
             throw new IllegalArgumentException(unknownErrorMessageSource.apply(componentType));
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index db3aeff0483f6..9c3d991080d26 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -120,7 +120,7 @@ public final class WriterConstants {
             DEF_BOOTSTRAP_METHOD.getDescriptor(), false);
     public static final Type DEF_BOOTSTRAP_DELEGATE_TYPE = Type.getType(DefBootstrap.class);
     public static final Method DEF_BOOTSTRAP_DELEGATE_METHOD = getAsmMethod(CallSite.class, "bootstrap", PainlessLookup.class,
-            MethodHandles.Lookup.class, String.class, MethodType.class, int.class, int.class, Object[].class);
+            Map.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, int.class, Object[].class);
 
     public static final Type DEF_UTIL_TYPE = Type.getType(Def.class);
     public static final Method DEF_TO_BOOLEAN         = getAsmMethod(boolean.class, "DefToboolean"       , Object.class);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
index f1db35636b41c..9279093cf31ae 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
@@ -75,7 +75,7 @@ public void recover(final LexerNoViableAltException lnvae) {
 
     @Override
     protected boolean isType(String name) {
-        return painlessLookup.isSimplePainlessType(name);
+        return painlessLookup.isValidCanonicalClassName(name);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java
index 2440fb45d4dfb..f87f8a134b8c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java
@@ -22,46 +22,55 @@
 public class PainlessCast {
 
     /** Create a standard cast with no boxing/unboxing. */
-    public static PainlessCast standard(Class<?> from, Class<?> to, boolean explicit) {
-        return new PainlessCast(from, to, explicit, null, null, null, null);
+    public static PainlessCast originalTypetoTargetType(Class<?> originalType, Class<?> targetType, boolean explicitCast) {
+        return new PainlessCast(originalType, targetType, explicitCast, null, null, null, null);
     }
 
-    /** Create a cast where the from type will be unboxed, and then the cast will be performed. */
-    public static PainlessCast unboxFrom(Class<?> from, Class<?> to, boolean explicit, Class<?> unboxFrom) {
-        return new PainlessCast(from, to, explicit, unboxFrom, null, null, null);
+    /** Create a cast where the original type will be unboxed, and then the cast will be performed. */
+    public static PainlessCast unboxOriginalType(
+            Class<?> originalType, Class<?> targetType, boolean explicitCast, Class<?> unboxOriginalType) {
+
+        return new PainlessCast(originalType, targetType, explicitCast, unboxOriginalType, null, null, null);
     }
 
-    /** Create a cast where the to type will be unboxed, and then the cast will be performed. */
-    public static PainlessCast unboxTo(Class<?> from, Class<?> to, boolean explicit, Class<?> unboxTo) {
-        return new PainlessCast(from, to, explicit, null, unboxTo, null, null);
+    /** Create a cast where the target type will be unboxed, and then the cast will be performed. */
+    public static PainlessCast unboxTargetType(
+            Class<?> originalType, Class<?> targetType, boolean explicitCast, Class<?> unboxTargetType) {
+
+        return new PainlessCast(originalType, targetType, explicitCast, null, unboxTargetType, null, null);
     }
 
-    /** Create a cast where the from type will be boxed, and then the cast will be performed. */
-    public static PainlessCast boxFrom(Class<?> from, Class<?> to, boolean explicit, Class<?> boxFrom) {
-        return new PainlessCast(from, to, explicit, null, null, boxFrom, null);
+    /** Create a cast where the original type will be boxed, and then the cast will be performed. */
+    public static PainlessCast boxOriginalType(
+            Class<?> originalType, Class<?> targetType, boolean explicitCast, Class<?> boxOriginalType) {
+
+        return new PainlessCast(originalType, targetType, explicitCast, null, null, boxOriginalType, null);
     }
 
-    /** Create a cast where the to type will be boxed, and then the cast will be performed. */
-    public static PainlessCast boxTo(Class<?> from, Class<?> to, boolean explicit, Class<?> boxTo) {
-        return new PainlessCast(from, to, explicit, null, null, null, boxTo);
+    /** Create a cast where the target type will be boxed, and then the cast will be performed. */
+    public static PainlessCast boxTargetType(
+            Class<?> originalType, Class<?> targetType, boolean explicitCast, Class<?> boxTargetType) {
+
+        return new PainlessCast(originalType, targetType, explicitCast, null, null, null, boxTargetType);
     }
 
-    public final Class<?> from;
-    public final Class<?> to;
-    public final boolean explicit;
-    public final Class<?> unboxFrom;
-    public final Class<?> unboxTo;
-    public final Class<?> boxFrom;
-    public final Class<?> boxTo;
+    public final Class<?> originalType;
+    public final Class<?> targetType;
+    public final boolean explicitCast;
+    public final Class<?> unboxOriginalType;
+    public final Class<?> unboxTargetType;
+    public final Class<?> boxOriginalType;
+    public final Class<?> boxTargetType;
+
+    private PainlessCast(Class<?> originalType, Class<?> targetType, boolean explicitCast,
+            Class<?> unboxOriginalType, Class<?> unboxTargetType, Class<?> boxOriginalType, Class<?> boxTargetType) {
 
-    private PainlessCast(Class<?> from, Class<?> to, boolean explicit,
-                         Class<?> unboxFrom, Class<?> unboxTo, Class<?> boxFrom, Class<?> boxTo) {
-        this.from = from;
-        this.to = to;
-        this.explicit = explicit;
-        this.unboxFrom = unboxFrom;
-        this.unboxTo = unboxTo;
-        this.boxFrom = boxFrom;
-        this.boxTo = boxTo;
+        this.originalType = originalType;
+        this.targetType = targetType;
+        this.explicitCast = explicitCast;
+        this.unboxOriginalType = unboxOriginalType;
+        this.unboxTargetType = unboxTargetType;
+        this.boxOriginalType = boxOriginalType;
+        this.boxTargetType = boxTargetType;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
index 24dcf0ebdba87..50bb79dcfbdf5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
@@ -24,7 +24,8 @@
 import java.util.Map;
 
 public final class PainlessClass {
-    public final Map<String, PainlessMethod> constructors;
+    public final Map<String, PainlessConstructor> constructors;
+
     public final Map<String, PainlessMethod> staticMethods;
     public final Map<String, PainlessMethod> methods;
 
@@ -34,15 +35,16 @@ public final class PainlessClass {
     public final Map<String, MethodHandle> getterMethodHandles;
     public final Map<String, MethodHandle> setterMethodHandles;
 
-    public final PainlessMethod functionalMethod;
+    public final PainlessMethod functionalInterfaceMethod;
 
-    PainlessClass(Map<String, PainlessMethod> constructors,
+    PainlessClass(Map<String, PainlessConstructor> constructors,
             Map<String, PainlessMethod> staticMethods, Map<String, PainlessMethod> methods,
             Map<String, PainlessField> staticFields, Map<String, PainlessField> fields,
             Map<String, MethodHandle> getterMethodHandles, Map<String, MethodHandle> setterMethodHandles,
-            PainlessMethod functionalMethod) {
+            PainlessMethod functionalInterfaceMethod) {
 
         this.constructors = Collections.unmodifiableMap(constructors);
+
         this.staticMethods = Collections.unmodifiableMap(staticMethods);
         this.methods = Collections.unmodifiableMap(methods);
 
@@ -52,6 +54,6 @@ public final class PainlessClass {
         this.getterMethodHandles = Collections.unmodifiableMap(getterMethodHandles);
         this.setterMethodHandles = Collections.unmodifiableMap(setterMethodHandles);
 
-        this.functionalMethod = functionalMethod;
+        this.functionalInterfaceMethod = functionalInterfaceMethod;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
index 2f41ed5dca8f1..a61215e9ed749 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
@@ -24,7 +24,8 @@
 import java.util.Map;
 
 final class PainlessClassBuilder {
-    final Map<String, PainlessMethod> constructors;
+    final Map<String, PainlessConstructor> constructors;
+
     final Map<String, PainlessMethod> staticMethods;
     final Map<String, PainlessMethod> methods;
 
@@ -34,10 +35,11 @@ final class PainlessClassBuilder {
     final Map<String, MethodHandle> getterMethodHandles;
     final Map<String, MethodHandle> setterMethodHandles;
 
-    PainlessMethod functionalMethod;
+    PainlessMethod functionalInterfaceMethod;
 
     PainlessClassBuilder() {
         constructors = new HashMap<>();
+
         staticMethods = new HashMap<>();
         methods = new HashMap<>();
 
@@ -47,11 +49,11 @@ final class PainlessClassBuilder {
         getterMethodHandles = new HashMap<>();
         setterMethodHandles = new HashMap<>();
 
-        functionalMethod = null;
+        functionalInterfaceMethod = null;
     }
 
     PainlessClass build() {
         return new PainlessClass(constructors, staticMethods, methods, staticFields, fields,
-                getterMethodHandles, setterMethodHandles, functionalMethod);
+                getterMethodHandles, setterMethodHandles, functionalInterfaceMethod);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessConstructor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessConstructor.java
new file mode 100644
index 0000000000000..76597c1a29d65
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessConstructor.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.lookup;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodType;
+import java.lang.reflect.Constructor;
+import java.util.List;
+
+public class PainlessConstructor {
+    public final Constructor<?> javaConstructor;
+    public final List<Class<?>> typeParameters;
+    public final MethodHandle methodHandle;
+    public final MethodType methodType;
+
+    PainlessConstructor(Constructor<?> javaConstructor, List<Class<?>> typeParameters, MethodHandle methodHandle, MethodType methodType) {
+        this.javaConstructor = javaConstructor;
+        this.typeParameters = typeParameters;
+        this.methodHandle = methodHandle;
+        this.methodType = methodType;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java
index f316e1438ecb9..a55d6c3730ebd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java
@@ -20,24 +20,20 @@
 package org.elasticsearch.painless.lookup;
 
 import java.lang.invoke.MethodHandle;
+import java.lang.reflect.Field;
 
 public final class PainlessField {
-    public final String name;
-    public final Class<?> target;
-    public final Class<?> clazz;
-    public final String javaName;
-    public final int modifiers;
-    public final MethodHandle getter;
-    public final MethodHandle setter;
+    public final Field javaField;
+    public final Class<?> typeParameter;
 
-    PainlessField(String name, String javaName, Class<?> target, Class<?> clazz, int modifiers,
-                  MethodHandle getter, MethodHandle setter) {
-        this.name = name;
-        this.javaName = javaName;
-        this.target = target;
-        this.clazz = clazz;
-        this.modifiers = modifiers;
-        this.getter = getter;
-        this.setter = setter;
+    public final MethodHandle getterMethodHandle;
+    public final MethodHandle setterMethodHandle;
+
+    PainlessField(Field javaField, Class<?> typeParameter, MethodHandle getterMethodHandle, MethodHandle setterMethodHandle) {
+        this.javaField = javaField;
+        this.typeParameter = typeParameter;
+
+        this.getterMethodHandle = getterMethodHandle;
+        this.setterMethodHandle = setterMethodHandle;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
index 67c04498a5894..16b8ac14f14f2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
@@ -19,41 +19,221 @@
 
 package org.elasticsearch.painless.lookup;
 
-import java.util.Collection;
+import java.lang.invoke.MethodHandle;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
 
-/**
- * The entire API for Painless.  Also used as a whitelist for checking for legal
- * methods and fields during at both compile-time and runtime.
- */
-public final class PainlessLookup {
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessConstructorKey;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessFieldKey;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToBoxedType;
 
-    public Collection<Class<?>> getStructs() {
-        return classesToPainlessClasses.keySet();
-    }
+public final class PainlessLookup {
 
     private final Map<String, Class<?>> canonicalClassNamesToClasses;
     private final Map<Class<?>, PainlessClass> classesToPainlessClasses;
 
     PainlessLookup(Map<String, Class<?>> canonicalClassNamesToClasses, Map<Class<?>, PainlessClass> classesToPainlessClasses) {
+        Objects.requireNonNull(canonicalClassNamesToClasses);
+        Objects.requireNonNull(classesToPainlessClasses);
+
         this.canonicalClassNamesToClasses = Collections.unmodifiableMap(canonicalClassNamesToClasses);
         this.classesToPainlessClasses = Collections.unmodifiableMap(classesToPainlessClasses);
     }
 
-    public Class<?> getClassFromBinaryName(String painlessType) {
-        return canonicalClassNamesToClasses.get(painlessType.replace('$', '.'));
+    public boolean isValidCanonicalClassName(String canonicalClassName) {
+        Objects.requireNonNull(canonicalClassName);
+
+        return canonicalClassNamesToClasses.containsKey(canonicalClassName);
+    }
+
+    public Class<?> canonicalTypeNameToType(String canonicalTypeName) {
+        Objects.requireNonNull(canonicalTypeName);
+
+        return PainlessLookupUtility.canonicalTypeNameToType(canonicalTypeName, canonicalClassNamesToClasses);
+    }
+
+    public Set<Class<?>> getClasses() {
+        return classesToPainlessClasses.keySet();
+    }
+
+    public PainlessClass lookupPainlessClass(Class<?> targetClass) {
+        return classesToPainlessClasses.get(targetClass);
+    }
+
+    public PainlessConstructor lookupPainlessConstructor(String targetCanonicalClassName, int constructorArity) {
+        Objects.requireNonNull(targetCanonicalClassName);
+
+        Class<?> targetClass = canonicalTypeNameToType(targetCanonicalClassName);
+
+        if (targetClass == null) {
+            return null;
+        }
+
+        return lookupPainlessConstructor(targetClass, constructorArity);
+    }
+
+    public PainlessConstructor lookupPainlessConstructor(Class<?> targetClass, int constructorArity) {
+        Objects.requireNonNull(targetClass);
+
+        PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetClass);
+        String painlessConstructorKey = buildPainlessConstructorKey(constructorArity);
+
+        if (targetPainlessClass == null) {
+            return null;
+        }
+
+        PainlessConstructor painlessConstructor = targetPainlessClass.constructors.get(painlessConstructorKey);
+
+        if (painlessConstructor == null) {
+            return null;
+        }
+
+        return painlessConstructor;
+    }
+
+    public PainlessMethod lookupPainlessMethod(String targetCanonicalClassName, boolean isStatic, String methodName, int methodArity) {
+        Objects.requireNonNull(targetCanonicalClassName);
+
+        Class<?> targetClass = canonicalTypeNameToType(targetCanonicalClassName);
+
+        if (targetClass == null) {
+            return null;
+        }
+
+        return lookupPainlessMethod(targetClass, isStatic, methodName, methodArity);
+    }
+
+    public PainlessMethod lookupPainlessMethod(Class<?> targetClass, boolean isStatic, String methodName, int methodArity) {
+        Objects.requireNonNull(targetClass);
+        Objects.requireNonNull(methodName);
+
+        if (targetClass.isPrimitive()) {
+            targetClass = typeToBoxedType(targetClass);
+        }
+
+        PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetClass);
+        String painlessMethodKey = buildPainlessMethodKey(methodName, methodArity);
+
+        if (targetPainlessClass == null) {
+            return null;
+        }
+
+        return isStatic ?
+                targetPainlessClass.staticMethods.get(painlessMethodKey) :
+                targetPainlessClass.methods.get(painlessMethodKey);
+    }
+
+    public PainlessField lookupPainlessField(String targetCanonicalClassName, boolean isStatic, String fieldName) {
+        Objects.requireNonNull(targetCanonicalClassName);
+
+        Class<?> targetClass = canonicalTypeNameToType(targetCanonicalClassName);
+
+        if (targetClass == null) {
+            return null;
+        }
+
+        return lookupPainlessField(targetClass, isStatic, fieldName);
+    }
+
+    public PainlessField lookupPainlessField(Class<?> targetClass, boolean isStatic, String fieldName) {
+        Objects.requireNonNull(targetClass);
+        Objects.requireNonNull(fieldName);
+
+        PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetClass);
+        String painlessFieldKey = buildPainlessFieldKey(fieldName);
+
+        if (targetPainlessClass == null) {
+            return null;
+        }
+
+        PainlessField painlessField = isStatic ?
+                targetPainlessClass.staticFields.get(painlessFieldKey) :
+                targetPainlessClass.fields.get(painlessFieldKey);
+
+        if (painlessField == null) {
+            return null;
+        }
+
+        return painlessField;
+    }
+
+    public PainlessMethod lookupFunctionalInterfacePainlessMethod(Class<?> targetClass) {
+        PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetClass);
+
+        if (targetPainlessClass == null) {
+            return null;
+        }
+
+        return targetPainlessClass.functionalInterfaceMethod;
+    }
+
+    public PainlessMethod lookupRuntimePainlessMethod(Class<?> originalTargetClass, String methodName, int methodArity) {
+        Objects.requireNonNull(originalTargetClass);
+        Objects.requireNonNull(methodName);
+
+        String painlessMethodKey = buildPainlessMethodKey(methodName, methodArity);
+        Function<PainlessClass, PainlessMethod> objectLookup = targetPainlessClass -> targetPainlessClass.methods.get(painlessMethodKey);
+
+        return lookupRuntimePainlessObject(originalTargetClass, objectLookup);
     }
 
-    public boolean isSimplePainlessType(String painlessType) {
-        return canonicalClassNamesToClasses.containsKey(painlessType);
+    public MethodHandle lookupRuntimeGetterMethodHandle(Class<?> originalTargetClass, String getterName) {
+        Objects.requireNonNull(originalTargetClass);
+        Objects.requireNonNull(getterName);
+
+        Function<PainlessClass, MethodHandle> objectLookup = targetPainlessClass -> targetPainlessClass.getterMethodHandles.get(getterName);
+
+        return lookupRuntimePainlessObject(originalTargetClass, objectLookup);
     }
 
-    public PainlessClass getPainlessStructFromJavaClass(Class<?> clazz) {
-        return classesToPainlessClasses.get(clazz);
+    public MethodHandle lookupRuntimeSetterMethodHandle(Class<?> originalTargetClass, String setterName) {
+        Objects.requireNonNull(originalTargetClass);
+        Objects.requireNonNull(setterName);
+
+        Function<PainlessClass, MethodHandle> objectLookup = targetPainlessClass -> targetPainlessClass.setterMethodHandles.get(setterName);
+
+        return lookupRuntimePainlessObject(originalTargetClass, objectLookup);
     }
 
-    public Class<?> getJavaClassFromPainlessType(String painlessType) {
-        return PainlessLookupUtility.canonicalTypeNameToType(painlessType, canonicalClassNamesToClasses);
+    private <T> T lookupRuntimePainlessObject(Class<?> originalTargetClass, Function<PainlessClass, T> objectLookup) {
+        Class<?> currentTargetClass = originalTargetClass;
+
+        while (currentTargetClass != null) {
+            PainlessClass targetPainlessClass = classesToPainlessClasses.get(currentTargetClass);
+
+            if (targetPainlessClass != null) {
+                T painlessObject = objectLookup.apply(targetPainlessClass);
+
+                if (painlessObject != null) {
+                    return painlessObject;
+                }
+            }
+
+            currentTargetClass = currentTargetClass.getSuperclass();
+        }
+
+        currentTargetClass = originalTargetClass;
+
+        while (currentTargetClass != null) {
+            for (Class<?> targetInterface : currentTargetClass.getInterfaces()) {
+                PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetInterface);
+
+                if (targetPainlessClass != null) {
+                    T painlessObject = objectLookup.apply(targetPainlessClass);
+
+                    if (painlessObject != null) {
+                        return painlessObject;
+                    }
+                }
+            }
+
+            currentTargetClass = currentTargetClass.getSuperclass();
+        }
+
+        return null;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index 3675cc7cd0f20..0346867d35aa3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -27,6 +27,7 @@
 
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
@@ -39,15 +40,47 @@
 import java.util.Objects;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.painless.lookup.PainlessLookupUtility.CONSTRUCTOR_NAME;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.DEF_CLASS_NAME;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessConstructorKey;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessFieldKey;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToJavaType;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typesToCanonicalTypeNames;
 
-public class PainlessLookupBuilder {
+public final class PainlessLookupBuilder {
+
+    private static class PainlessConstructorCacheKey {
+
+        private final Class<?> targetType;
+        private final List<Class<?>> typeParameters;
+
+        private PainlessConstructorCacheKey(Class<?> targetType, List<Class<?>> typeParameters) {
+            this.targetType = targetType;
+            this.typeParameters = Collections.unmodifiableList(typeParameters);
+        }
+
+        @Override
+        public boolean equals(Object object) {
+            if (this == object) {
+                return true;
+            }
+
+            if (object == null || getClass() != object.getClass()) {
+                return false;
+            }
+
+            PainlessConstructorCacheKey that = (PainlessConstructorCacheKey)object;
+
+            return Objects.equals(targetType, that.targetType) &&
+                   Objects.equals(typeParameters, that.typeParameters);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(targetType, typeParameters);
+        }
+    }
 
     private static class PainlessMethodCacheKey {
 
@@ -119,8 +152,9 @@ public int hashCode() {
         }
     }
 
-    private static final Map<PainlessMethodCacheKey, PainlessMethod> painlessMethodCache = new HashMap<>();
-    private static final Map<PainlessFieldCacheKey,  PainlessField>  painlessFieldCache  = new HashMap<>();
+    private static final Map<PainlessConstructorCacheKey, PainlessConstructor> painlessConstuctorCache = new HashMap<>();
+    private static final Map<PainlessMethodCacheKey,      PainlessMethod>      painlessMethodCache     = new HashMap<>();
+    private static final Map<PainlessFieldCacheKey,       PainlessField>       painlessFieldCache      = new HashMap<>();
 
     private static final Pattern CLASS_NAME_PATTERN  = Pattern.compile("^[_a-zA-Z][._a-zA-Z0-9]*$");
     private static final Pattern METHOD_NAME_PATTERN = Pattern.compile("^[_a-zA-Z][_a-zA-Z0-9]*$");
@@ -132,35 +166,35 @@ public static PainlessLookup buildFromWhitelists(List<Whitelist> whitelists) {
 
         try {
             for (Whitelist whitelist : whitelists) {
-                for (WhitelistClass whitelistClass : whitelist.whitelistStructs) {
+                for (WhitelistClass whitelistClass : whitelist.whitelistClasses) {
                     origin = whitelistClass.origin;
                     painlessLookupBuilder.addPainlessClass(
-                            whitelist.javaClassLoader, whitelistClass.javaClassName, whitelistClass.onlyFQNJavaClassName == false);
+                            whitelist.classLoader, whitelistClass.javaClassName, whitelistClass.noImport == false);
                 }
             }
 
             for (Whitelist whitelist : whitelists) {
-                for (WhitelistClass whitelistClass : whitelist.whitelistStructs) {
+                for (WhitelistClass whitelistClass : whitelist.whitelistClasses) {
                     String targetCanonicalClassName = whitelistClass.javaClassName.replace('$', '.');
 
                     for (WhitelistConstructor whitelistConstructor : whitelistClass.whitelistConstructors) {
                         origin = whitelistConstructor.origin;
                         painlessLookupBuilder.addPainlessConstructor(
-                                targetCanonicalClassName, whitelistConstructor.painlessParameterTypeNames);
+                                targetCanonicalClassName, whitelistConstructor.canonicalTypeNameParameters);
                     }
 
                     for (WhitelistMethod whitelistMethod : whitelistClass.whitelistMethods) {
                         origin = whitelistMethod.origin;
                         painlessLookupBuilder.addPainlessMethod(
-                                whitelist.javaClassLoader, targetCanonicalClassName, whitelistMethod.javaAugmentedClassName,
-                                whitelistMethod.javaMethodName, whitelistMethod.painlessReturnTypeName,
-                                whitelistMethod.painlessParameterTypeNames);
+                                whitelist.classLoader, targetCanonicalClassName, whitelistMethod.augmentedCanonicalClassName,
+                                whitelistMethod.methodName, whitelistMethod.returnCanonicalTypeName,
+                                whitelistMethod.canonicalTypeNameParameters);
                     }
 
                     for (WhitelistField whitelistField : whitelistClass.whitelistFields) {
                         origin = whitelistField.origin;
                         painlessLookupBuilder.addPainlessField(
-                                targetCanonicalClassName, whitelistField.javaFieldName, whitelistField.painlessFieldTypeName);
+                                targetCanonicalClassName, whitelistField.fieldName, whitelistField.canonicalTypeNameParameter);
                     }
                 }
             }
@@ -186,8 +220,12 @@ private Class<?> canonicalTypeNameToType(String canonicalTypeName) {
         return PainlessLookupUtility.canonicalTypeNameToType(canonicalTypeName, canonicalClassNamesToClasses);
     }
 
-    private void validateType(Class<?> type) {
-        PainlessLookupUtility.validateType(type, classesToPainlessClassBuilders.keySet());
+    private boolean isValidType(Class<?> type) {
+        while (type.getComponentType() != null) {
+            type = type.getComponentType();
+        }
+
+        return classesToPainlessClassBuilders.containsKey(type);
     }
 
     public void addPainlessClass(ClassLoader classLoader, String javaClassName, boolean importClassName) {
@@ -277,27 +315,28 @@ public void addPainlessClass(Class<?> clazz, boolean importClassName) {
         }
     }
 
-    public void addPainlessConstructor(String targetCanonicalClassName, List<String> typeNameParameters) {
+    public void addPainlessConstructor(String targetCanonicalClassName, List<String> canonicalTypeNameParameters) {
         Objects.requireNonNull(targetCanonicalClassName);
-        Objects.requireNonNull(typeNameParameters);
+        Objects.requireNonNull(canonicalTypeNameParameters);
 
         Class<?> targetClass = canonicalClassNamesToClasses.get(targetCanonicalClassName);
 
         if (targetClass == null) {
             throw new IllegalArgumentException("target class [" + targetCanonicalClassName + "] not found" +
-                    "for constructor [[" + targetCanonicalClassName + "], " + typeNameParameters  + "]");
+                    "for constructor [[" + targetCanonicalClassName + "], " + canonicalTypeNameParameters  + "]");
         }
 
-        List<Class<?>> typeParameters = new ArrayList<>(typeNameParameters.size());
+        List<Class<?>> typeParameters = new ArrayList<>(canonicalTypeNameParameters.size());
 
-        for (String typeNameParameter : typeNameParameters) {
-            try {
-                Class<?> typeParameter = canonicalTypeNameToType(typeNameParameter);
-                typeParameters.add(typeParameter);
-            } catch (IllegalArgumentException iae) {
-                throw new IllegalArgumentException("type parameter [" + typeNameParameter + "] not found " +
-                        "for constructor [[" + targetCanonicalClassName + "], " + typeNameParameters  + "]", iae);
+        for (String canonicalTypeNameParameter : canonicalTypeNameParameters) {
+            Class<?> typeParameter = canonicalTypeNameToType(canonicalTypeNameParameter);
+
+            if (typeParameter == null) {
+                throw new IllegalArgumentException("type parameter [" + canonicalTypeNameParameter + "] not found " +
+                        "for constructor [[" + targetCanonicalClassName + "], " + canonicalTypeNameParameters  + "]");
             }
+
+            typeParameters.add(typeParameter);
         }
 
         addPainlessConstructor(targetClass, typeParameters);
@@ -323,11 +362,9 @@ public void addPainlessConstructor(Class<?> targetClass, List<Class<?>> typePara
         List<Class<?>> javaTypeParameters = new ArrayList<>(typeParametersSize);
 
         for (Class<?> typeParameter : typeParameters) {
-            try {
-                validateType(typeParameter);
-            } catch (IllegalArgumentException iae) {
+            if (isValidType(typeParameter) == false) {
                 throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(typeParameter) + "] not found " +
-                        "for constructor [[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "]", iae);
+                        "for constructor [[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "]");
             }
 
             javaTypeParameters.add(typeToJavaType(typeParameter));
@@ -342,11 +379,10 @@ public void addPainlessConstructor(Class<?> targetClass, List<Class<?>> typePara
                     "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", nsme);
         }
 
-        String painlessMethodKey = buildPainlessMethodKey(CONSTRUCTOR_NAME, typeParametersSize);
-        PainlessMethod painlessConstructor = painlessClassBuilder.constructors.get(painlessMethodKey);
+        String painlessConstructorKey = buildPainlessConstructorKey(typeParametersSize);
+        PainlessConstructor painlessConstructor = painlessClassBuilder.constructors.get(painlessConstructorKey);
 
         if (painlessConstructor == null) {
-            org.objectweb.asm.commons.Method asmConstructor = org.objectweb.asm.commons.Method.getMethod(javaConstructor);
             MethodHandle methodHandle;
 
             try {
@@ -356,35 +392,36 @@ public void addPainlessConstructor(Class<?> targetClass, List<Class<?>> typePara
                         "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", iae);
             }
 
-            painlessConstructor = painlessMethodCache.computeIfAbsent(
-                    new PainlessMethodCacheKey(targetClass, CONSTRUCTOR_NAME, typeParameters),
-                    key -> new PainlessMethod(CONSTRUCTOR_NAME, targetClass, null, void.class, typeParameters,
-                                              asmConstructor, javaConstructor.getModifiers(), methodHandle)
+            MethodType methodType = methodHandle.type();
+
+            painlessConstructor = painlessConstuctorCache.computeIfAbsent(
+                    new PainlessConstructorCacheKey(targetClass, typeParameters),
+                    key -> new PainlessConstructor(javaConstructor, typeParameters, methodHandle, methodType)
             );
 
-            painlessClassBuilder.constructors.put(painlessMethodKey, painlessConstructor);
-        } else if (painlessConstructor.arguments.equals(typeParameters) == false){
+            painlessClassBuilder.constructors.put(painlessConstructorKey, painlessConstructor);
+        } else if (painlessConstructor.typeParameters.equals(typeParameters) == false){
             throw new IllegalArgumentException("cannot have constructors " +
                     "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "] and " +
-                    "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(painlessConstructor.arguments) + "] " +
+                    "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(painlessConstructor.typeParameters) + "] " +
                     "with the same arity and different type parameters");
         }
     }
 
     public void addPainlessMethod(ClassLoader classLoader, String targetCanonicalClassName, String augmentedCanonicalClassName,
-            String methodName, String returnCanonicalTypeName, List<String> typeNameParameters) {
+            String methodName, String returnCanonicalTypeName, List<String> canonicalTypeNameParameters) {
 
         Objects.requireNonNull(classLoader);
         Objects.requireNonNull(targetCanonicalClassName);
         Objects.requireNonNull(methodName);
         Objects.requireNonNull(returnCanonicalTypeName);
-        Objects.requireNonNull(typeNameParameters);
+        Objects.requireNonNull(canonicalTypeNameParameters);
 
         Class<?> targetClass = canonicalClassNamesToClasses.get(targetCanonicalClassName);
 
         if (targetClass == null) {
             throw new IllegalArgumentException("target class [" + targetCanonicalClassName + "] not found for method " +
-                    "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]");
+                    "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]");
         }
 
         Class<?> augmentedClass = null;
@@ -394,29 +431,28 @@ public void addPainlessMethod(ClassLoader classLoader, String targetCanonicalCla
                 augmentedClass = Class.forName(augmentedCanonicalClassName, true, classLoader);
             } catch (ClassNotFoundException cnfe) {
                 throw new IllegalArgumentException("augmented class [" + augmentedCanonicalClassName + "] not found for method " +
-                        "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]", cnfe);
+                        "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]", cnfe);
             }
         }
 
-        List<Class<?>> typeParameters = new ArrayList<>(typeNameParameters.size());
+        List<Class<?>> typeParameters = new ArrayList<>(canonicalTypeNameParameters.size());
 
-        for (String typeNameParameter : typeNameParameters) {
-            try {
-                Class<?> typeParameter = canonicalTypeNameToType(typeNameParameter);
-                typeParameters.add(typeParameter);
-            } catch (IllegalArgumentException iae) {
-                throw new IllegalArgumentException("parameter type [" + typeNameParameter + "] not found for method " +
-                        "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]", iae);
+        for (String canonicalTypeNameParameter : canonicalTypeNameParameters) {
+            Class<?> typeParameter = canonicalTypeNameToType(canonicalTypeNameParameter);
+
+            if (typeParameter == null) {
+                throw new IllegalArgumentException("parameter type [" + canonicalTypeNameParameter + "] not found for method " +
+                        "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]");
             }
+
+            typeParameters.add(typeParameter);
         }
 
-        Class<?> returnType;
+        Class<?> returnType = canonicalTypeNameToType(returnCanonicalTypeName);
 
-        try {
-            returnType = canonicalTypeNameToType(returnCanonicalTypeName);
-        } catch (IllegalArgumentException iae) {
+        if (returnType == null) {
             throw new IllegalArgumentException("parameter type [" + returnCanonicalTypeName + "] not found for method " +
-                    "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]", iae);
+                    "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]");
         }
 
         addPainlessMethod(targetClass, augmentedClass, methodName, returnType, typeParameters);
@@ -456,22 +492,18 @@ public void addPainlessMethod(Class<?> targetClass, Class<?> augmentedClass, Str
         }
 
         for (Class<?> typeParameter : typeParameters) {
-            try {
-                validateType(typeParameter);
-            } catch (IllegalArgumentException iae) {
+            if (isValidType(typeParameter) == false) {
                 throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(typeParameter) + "] " +
                         "not found for method [[" + targetCanonicalClassName + "], [" + methodName + "], " +
-                        typesToCanonicalTypeNames(typeParameters) + "]", iae);
+                        typesToCanonicalTypeNames(typeParameters) + "]");
             }
 
             javaTypeParameters.add(typeToJavaType(typeParameter));
         }
 
-        try {
-            validateType(returnType);
-        } catch (IllegalArgumentException iae) {
+        if (isValidType(returnType) == false) {
             throw new IllegalArgumentException("return type [" + typeToCanonicalTypeName(returnType) + "] not found for method " +
-                    "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]", iae);
+                    "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]");
         }
 
         Method javaMethod;
@@ -506,7 +538,6 @@ public void addPainlessMethod(Class<?> targetClass, Class<?> augmentedClass, Str
             PainlessMethod painlessMethod = painlessClassBuilder.staticMethods.get(painlessMethodKey);
 
             if (painlessMethod == null) {
-                org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
                 MethodHandle methodHandle;
 
                 try {
@@ -516,28 +547,27 @@ public void addPainlessMethod(Class<?> targetClass, Class<?> augmentedClass, Str
                             "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", iae);
                 }
 
+                MethodType methodType = methodHandle.type();
+
                 painlessMethod = painlessMethodCache.computeIfAbsent(
                         new PainlessMethodCacheKey(targetClass, methodName, typeParameters),
-                        key -> new PainlessMethod(methodName, targetClass, null, returnType,
-                                typeParameters, asmMethod, javaMethod.getModifiers(), methodHandle));
+                        key -> new PainlessMethod(javaMethod, targetClass, returnType, typeParameters, methodHandle, methodType));
 
                 painlessClassBuilder.staticMethods.put(painlessMethodKey, painlessMethod);
-            } else if ((painlessMethod.name.equals(methodName) && painlessMethod.rtn == returnType &&
-                    painlessMethod.arguments.equals(typeParameters)) == false) {
+            } else if (painlessMethod.returnType == returnType && painlessMethod.typeParameters.equals(typeParameters) == false) {
                 throw new IllegalArgumentException("cannot have static methods " +
                         "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
                         "[" + typeToCanonicalTypeName(returnType) + "], " +
                         typesToCanonicalTypeNames(typeParameters) + "] and " +
                         "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
-                        "[" + typeToCanonicalTypeName(painlessMethod.rtn) + "], " +
-                        typesToCanonicalTypeNames(painlessMethod.arguments) + "] " +
+                        "[" + typeToCanonicalTypeName(painlessMethod.returnType) + "], " +
+                        typesToCanonicalTypeNames(painlessMethod.typeParameters) + "] " +
                         "with the same arity and different return type or type parameters");
             }
         } else {
             PainlessMethod painlessMethod = painlessClassBuilder.staticMethods.get(painlessMethodKey);
 
             if (painlessMethod == null) {
-                org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
                 MethodHandle methodHandle;
 
                 if (augmentedClass == null) {
@@ -557,30 +587,30 @@ public void addPainlessMethod(Class<?> targetClass, Class<?> augmentedClass, Str
                     }
                 }
 
+                MethodType methodType = methodHandle.type();
+
                 painlessMethod = painlessMethodCache.computeIfAbsent(
                         new PainlessMethodCacheKey(targetClass, methodName, typeParameters),
-                        key -> new PainlessMethod(methodName, targetClass, augmentedClass, returnType,
-                                typeParameters, asmMethod, javaMethod.getModifiers(), methodHandle));
+                        key -> new PainlessMethod(javaMethod, targetClass, returnType, typeParameters, methodHandle, methodType));
 
                 painlessClassBuilder.methods.put(painlessMethodKey, painlessMethod);
-            } else if ((painlessMethod.name.equals(methodName) && painlessMethod.rtn == returnType &&
-                    painlessMethod.arguments.equals(typeParameters)) == false) {
+            } else if (painlessMethod.returnType == returnType && painlessMethod.typeParameters.equals(typeParameters) == false) {
                 throw new IllegalArgumentException("cannot have methods " +
                         "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
                         "[" + typeToCanonicalTypeName(returnType) + "], " +
                         typesToCanonicalTypeNames(typeParameters) + "] and " +
                         "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
-                        "[" + typeToCanonicalTypeName(painlessMethod.rtn) + "], " +
-                        typesToCanonicalTypeNames(painlessMethod.arguments) + "] " +
+                        "[" + typeToCanonicalTypeName(painlessMethod.returnType) + "], " +
+                        typesToCanonicalTypeNames(painlessMethod.typeParameters) + "] " +
                         "with the same arity and different return type or type parameters");
             }
         }
     }
 
-    public void addPainlessField(String targetCanonicalClassName, String fieldName, String typeNameParameter) {
+    public void addPainlessField(String targetCanonicalClassName, String fieldName, String canonicalTypeNameParameter) {
         Objects.requireNonNull(targetCanonicalClassName);
         Objects.requireNonNull(fieldName);
-        Objects.requireNonNull(typeNameParameter);
+        Objects.requireNonNull(canonicalTypeNameParameter);
 
         Class<?> targetClass = canonicalClassNamesToClasses.get(targetCanonicalClassName);
 
@@ -588,12 +618,10 @@ public void addPainlessField(String targetCanonicalClassName, String fieldName,
             throw new IllegalArgumentException("class [" + targetCanonicalClassName + "] not found");
         }
 
-        Class<?> typeParameter;
+        Class<?> typeParameter = canonicalTypeNameToType(canonicalTypeNameParameter);
 
-        try {
-            typeParameter = canonicalTypeNameToType(typeNameParameter);
-        } catch (IllegalArgumentException iae) {
-            throw new IllegalArgumentException("type parameter [" + typeNameParameter + "] not found " +
+        if (typeParameter == null) {
+            throw new IllegalArgumentException("type parameter [" + canonicalTypeNameParameter + "] not found " +
                     "for field [[" + targetCanonicalClassName + "], [" + fieldName + "]");
         }
 
@@ -624,11 +652,9 @@ public void addPainlessField(Class<?> targetClass, String fieldName, Class<?> ty
             throw new IllegalArgumentException("class [" + targetCanonicalClassName + "] not found");
         }
 
-        try {
-            validateType(typeParameter);
-        } catch (IllegalArgumentException iae) {
+        if (isValidType(typeParameter) == false) {
             throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(typeParameter) + "] not found " +
-                    "for field [[" + targetCanonicalClassName + "], [" + fieldName + "]", iae);
+                    "for field [[" + targetCanonicalClassName + "], [" + fieldName + "]");
         }
 
         Field javaField;
@@ -646,11 +672,20 @@ public void addPainlessField(Class<?> targetClass, String fieldName, Class<?> ty
                     "for field [[" + targetCanonicalClassName + "], [" + fieldName + "]");
         }
 
+        MethodHandle methodHandleGetter;
+
+        try {
+            methodHandleGetter = MethodHandles.publicLookup().unreflectGetter(javaField);
+        } catch (IllegalAccessException iae) {
+            throw new IllegalArgumentException(
+                    "getter method handle not found for field [[" + targetCanonicalClassName + "], [" + fieldName + "]]");
+        }
+
         String painlessFieldKey = buildPainlessFieldKey(fieldName);
 
         if (Modifier.isStatic(javaField.getModifiers())) {
             if (Modifier.isFinal(javaField.getModifiers()) == false) {
-                throw new IllegalArgumentException("static field [[" + targetCanonicalClassName + "]. [" + fieldName + "]] must be final");
+                throw new IllegalArgumentException("static field [[" + targetCanonicalClassName + "], [" + fieldName + "]] must be final");
             }
 
             PainlessField painlessField = painlessClassBuilder.staticFields.get(painlessFieldKey);
@@ -658,28 +693,18 @@ public void addPainlessField(Class<?> targetClass, String fieldName, Class<?> ty
             if (painlessField == null) {
                 painlessField = painlessFieldCache.computeIfAbsent(
                         new PainlessFieldCacheKey(targetClass, fieldName, typeParameter),
-                        key -> new PainlessField(fieldName, javaField.getName(), targetClass,
-                                typeParameter, javaField.getModifiers(), null, null));
+                        key -> new PainlessField(javaField, typeParameter, methodHandleGetter, null));
 
                 painlessClassBuilder.staticFields.put(painlessFieldKey, painlessField);
-            } else if (painlessField.clazz != typeParameter) {
+            } else if (painlessField.typeParameter != typeParameter) {
                 throw new IllegalArgumentException("cannot have static fields " +
                         "[[" + targetCanonicalClassName + "], [" + fieldName + "], [" +
                         typeToCanonicalTypeName(typeParameter) + "] and " +
-                        "[[" + targetCanonicalClassName + "], [" + painlessField.name + "], " +
-                        typeToCanonicalTypeName(painlessField.clazz) + "] " +
-                        "with the same and different type parameters");
+                        "[[" + targetCanonicalClassName + "], [" + painlessField.javaField.getName() + "], " +
+                        typeToCanonicalTypeName(painlessField.typeParameter) + "] " +
+                        "with the same name and different type parameters");
             }
         } else {
-            MethodHandle methodHandleGetter;
-
-            try {
-                methodHandleGetter = MethodHandles.publicLookup().unreflectGetter(javaField);
-            } catch (IllegalAccessException iae) {
-                throw new IllegalArgumentException(
-                        "getter method handle not found for field [[" + targetCanonicalClassName + "], [" + fieldName + "]]");
-            }
-
             MethodHandle methodHandleSetter;
 
             try {
@@ -694,17 +719,16 @@ public void addPainlessField(Class<?> targetClass, String fieldName, Class<?> ty
             if (painlessField == null) {
                 painlessField = painlessFieldCache.computeIfAbsent(
                         new PainlessFieldCacheKey(targetClass, painlessFieldKey, typeParameter),
-                        key -> new PainlessField(fieldName, javaField.getName(), targetClass,
-                                typeParameter, javaField.getModifiers(), methodHandleGetter, methodHandleSetter));
+                        key -> new PainlessField(javaField, typeParameter, methodHandleGetter, methodHandleSetter));
 
                 painlessClassBuilder.fields.put(fieldName, painlessField);
-            } else if (painlessField.clazz != typeParameter) {
+            } else if (painlessField.typeParameter != typeParameter) {
                 throw new IllegalArgumentException("cannot have fields " +
                         "[[" + targetCanonicalClassName + "], [" + fieldName + "], [" +
                         typeToCanonicalTypeName(typeParameter) + "] and " +
-                        "[[" + targetCanonicalClassName + "], [" + painlessField.name + "], " +
-                        typeToCanonicalTypeName(painlessField.clazz) + "] " +
-                        "with the same and different type parameters");
+                        "[[" + targetCanonicalClassName + "], [" + painlessField.javaField.getName() + "], " +
+                        typeToCanonicalTypeName(painlessField.typeParameter) + "] " +
+                        "with the same name and different type parameters");
             }
         }
     }
@@ -768,8 +792,8 @@ private void copyPainlessClassMembers(Class<?> originalClass, Class<?> targetCla
             PainlessMethod newPainlessMethod = painlessMethodEntry.getValue();
             PainlessMethod existingPainlessMethod = targetPainlessClassBuilder.methods.get(painlessMethodKey);
 
-            if (existingPainlessMethod == null || existingPainlessMethod.target != newPainlessMethod.target &&
-                    existingPainlessMethod.target.isAssignableFrom(newPainlessMethod.target)) {
+            if (existingPainlessMethod == null || existingPainlessMethod.targetClass != newPainlessMethod.targetClass &&
+                    existingPainlessMethod.targetClass.isAssignableFrom(newPainlessMethod.targetClass)) {
                 targetPainlessClassBuilder.methods.put(painlessMethodKey, newPainlessMethod);
             }
         }
@@ -779,8 +803,9 @@ private void copyPainlessClassMembers(Class<?> originalClass, Class<?> targetCla
             PainlessField newPainlessField = painlessFieldEntry.getValue();
             PainlessField existingPainlessField = targetPainlessClassBuilder.fields.get(painlessFieldKey);
 
-            if (existingPainlessField == null || existingPainlessField.target != newPainlessField.target &&
-                    existingPainlessField.target.isAssignableFrom(newPainlessField.target)) {
+            if (existingPainlessField == null ||
+                    existingPainlessField.javaField.getDeclaringClass() != newPainlessField.javaField.getDeclaringClass() &&
+                    existingPainlessField.javaField.getDeclaringClass().isAssignableFrom(newPainlessField.javaField.getDeclaringClass())) {
                 targetPainlessClassBuilder.fields.put(painlessFieldKey, newPainlessField);
             }
         }
@@ -794,27 +819,27 @@ private void cacheRuntimeHandles() {
 
     private void cacheRuntimeHandles(PainlessClassBuilder painlessClassBuilder) {
         for (PainlessMethod painlessMethod : painlessClassBuilder.methods.values()) {
-            String methodName = painlessMethod.name;
-            int typeParametersSize = painlessMethod.arguments.size();
+            String methodName = painlessMethod.javaMethod.getName();
+            int typeParametersSize = painlessMethod.typeParameters.size();
 
             if (typeParametersSize == 0 && methodName.startsWith("get") && methodName.length() > 3 &&
                     Character.isUpperCase(methodName.charAt(3))) {
                 painlessClassBuilder.getterMethodHandles.putIfAbsent(
-                        Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.handle);
+                        Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.methodHandle);
             } else if (typeParametersSize == 0 && methodName.startsWith("is") && methodName.length() > 2 &&
                     Character.isUpperCase(methodName.charAt(2))) {
                 painlessClassBuilder.getterMethodHandles.putIfAbsent(
-                        Character.toLowerCase(methodName.charAt(2)) + methodName.substring(3), painlessMethod.handle);
+                        Character.toLowerCase(methodName.charAt(2)) + methodName.substring(3), painlessMethod.methodHandle);
             } else if (typeParametersSize == 1 && methodName.startsWith("set") && methodName.length() > 3 &&
                     Character.isUpperCase(methodName.charAt(3))) {
                 painlessClassBuilder.setterMethodHandles.putIfAbsent(
-                        Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.handle);
+                        Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.methodHandle);
             }
         }
 
         for (PainlessField painlessField : painlessClassBuilder.fields.values()) {
-            painlessClassBuilder.getterMethodHandles.put(painlessField.name, painlessField.getter);
-            painlessClassBuilder.setterMethodHandles.put(painlessField.name, painlessField.setter);
+            painlessClassBuilder.getterMethodHandles.put(painlessField.javaField.getName(), painlessField.getterMethodHandle);
+            painlessClassBuilder.setterMethodHandles.put(painlessField.javaField.getName(), painlessField.setterMethodHandle);
         }
     }
 
@@ -844,7 +869,7 @@ private void setFunctionalInterfaceMethod(Class<?> targetClass, PainlessClassBui
             } else if (javaMethods.size() == 1) {
                 java.lang.reflect.Method javaMethod = javaMethods.get(0);
                 String painlessMethodKey = buildPainlessMethodKey(javaMethod.getName(), javaMethod.getParameterCount());
-                painlessClassBuilder.functionalMethod = painlessClassBuilder.methods.get(painlessMethodKey);
+                painlessClassBuilder.functionalInterfaceMethod = painlessClassBuilder.methods.get(painlessMethodKey);
             }
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
index 86d3f87663867..f2eb434516961 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.lookup;
 
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -101,45 +100,47 @@ public static Class<?> canonicalTypeNameToType(String canonicalTypeName, Map<Str
                     canonicalTypeName.charAt(arrayIndex++) == ']') {
                     ++arrayDimensions;
                 } else {
-                    throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found");
+                    return null;
                 }
             }
 
             canonicalTypeName = canonicalTypeName.substring(0, canonicalTypeName.indexOf('['));
             type = canonicalClassNamesToClasses.get(canonicalTypeName);
 
-            char arrayBraces[] = new char[arrayDimensions];
-            Arrays.fill(arrayBraces, '[');
-            String javaTypeName = new String(arrayBraces);
-
-            if (type == boolean.class) {
-                javaTypeName += "Z";
-            } else if (type == byte.class) {
-                javaTypeName += "B";
-            } else if (type == short.class) {
-                javaTypeName += "S";
-            } else if (type == char.class) {
-                javaTypeName += "C";
-            } else if (type == int.class) {
-                javaTypeName += "I";
-            } else if (type == long.class) {
-                javaTypeName += "J";
-            } else if (type == float.class) {
-                javaTypeName += "F";
-            } else if (type == double.class) {
-                javaTypeName += "D";
-            } else {
-                javaTypeName += "L" + type.getName() + ";";
-            }
+            if (type != null) {
+                char arrayBraces[] = new char[arrayDimensions];
+                Arrays.fill(arrayBraces, '[');
+                String javaTypeName = new String(arrayBraces);
+
+                if (type == boolean.class) {
+                    javaTypeName += "Z";
+                } else if (type == byte.class) {
+                    javaTypeName += "B";
+                } else if (type == short.class) {
+                    javaTypeName += "S";
+                } else if (type == char.class) {
+                    javaTypeName += "C";
+                } else if (type == int.class) {
+                    javaTypeName += "I";
+                } else if (type == long.class) {
+                    javaTypeName += "J";
+                } else if (type == float.class) {
+                    javaTypeName += "F";
+                } else if (type == double.class) {
+                    javaTypeName += "D";
+                } else {
+                    javaTypeName += "L" + type.getName() + ";";
+                }
 
-            try {
-                return Class.forName(javaTypeName);
-            } catch (ClassNotFoundException cnfe) {
-                throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found", cnfe);
+                try {
+                    return Class.forName(javaTypeName);
+                } catch (ClassNotFoundException cnfe) {
+                    throw new IllegalStateException("internal error", cnfe);
+                }
             }
         }
 
-        throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found");
+        return null;
     }
 
     /**
@@ -152,7 +153,9 @@ public static String typeToCanonicalTypeName(Class<?> type) {
 
         String canonicalTypeName = type.getCanonicalName();
 
-        if (canonicalTypeName.startsWith(def.class.getCanonicalName())) {
+        if (canonicalTypeName == null) {
+            canonicalTypeName = ANONYMOUS_CLASS_NAME;
+        } else if (canonicalTypeName.startsWith(def.class.getCanonicalName())) {
             canonicalTypeName = canonicalTypeName.replace(def.class.getCanonicalName(), DEF_CLASS_NAME);
         }
 
@@ -252,22 +255,6 @@ public static Class<?> typeToJavaType(Class<?> type) {
         return type;
     }
 
-    /**
-     * Ensures a type exists based on the terminology specified as part of {@link PainlessLookupUtility}.  Throws an
-     * {@link IllegalArgumentException} if the type does not exist.
-     */
-    public static void validateType(Class<?> type, Collection<Class<?>> classes) {
-        String canonicalTypeName = typeToCanonicalTypeName(type);
-
-        while (type.getComponentType() != null) {
-            type = type.getComponentType();
-        }
-
-        if (classes.contains(type) == false) {
-            throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found");
-        }
-    }
-
     /**
      * Converts a type  to its boxed type equivalent if one exists based on the terminology specified as part of
      * {@link PainlessLookupUtility}. Otherwise, this behaves as an identity function.
@@ -336,6 +323,13 @@ public static boolean isConstantType(Class<?> type) {
                type == String.class;
     }
 
+    /**
+     * Constructs a painless constructor key used to lookup painless constructors from a painless class.
+     */
+    public static String buildPainlessConstructorKey(int constructorArity) {
+        return CONSTRUCTOR_NAME + "/" + constructorArity;
+    }
+
     /**
      * Constructs a painless method key used to lookup painless methods from a painless class.
      */
@@ -350,6 +344,11 @@ public static String buildPainlessFieldKey(String fieldName) {
         return fieldName;
     }
 
+    /**
+     * The name for an anonymous class.
+     */
+    public static final String ANONYMOUS_CLASS_NAME = "$anonymous";
+
     /**
      * The def type name as specified in the source for a script.
      */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
index 3321de94a267f..9dd143a402865 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
@@ -19,113 +19,28 @@
 
 package org.elasticsearch.painless.lookup;
 
-import org.elasticsearch.painless.MethodWriter;
-import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.Type;
-
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodType;
-import java.lang.reflect.Modifier;
+import java.lang.reflect.Method;
 import java.util.Collections;
 import java.util.List;
 
 public class PainlessMethod {
-    public final String name;
-    public final Class<?> target;
-    public final Class<?> augmentation;
-    public final Class<?> rtn;
-    public final List<Class<?>> arguments;
-    public final org.objectweb.asm.commons.Method method;
-    public final int modifiers;
-    public final MethodHandle handle;
-
-    public PainlessMethod(String name, Class<?> target, Class<?> augmentation, Class<?> rtn, List<Class<?>> arguments,
-                          org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle) {
-        this.name = name;
-        this.augmentation = augmentation;
-        this.target = target;
-        this.rtn = rtn;
-        this.arguments = Collections.unmodifiableList(arguments);
-        this.method = method;
-        this.modifiers = modifiers;
-        this.handle = handle;
-    }
-
-    /**
-     * Returns MethodType for this method.
-     * <p>
-     * This works even for user-defined Methods (where the MethodHandle is null).
-     */
-    public MethodType getMethodType() {
-        // we have a methodhandle already (e.g. whitelisted class)
-        // just return its type
-        if (handle != null) {
-            return handle.type();
-        }
-        // otherwise compute it
-        final Class<?> params[];
-        final Class<?> returnValue;
-        if (augmentation != null) {
-            // static method disguised as virtual/interface method
-            params = new Class<?>[1 + arguments.size()];
-            params[0] = augmentation;
-            for (int i = 0; i < arguments.size(); i++) {
-                params[i + 1] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
-            }
-            returnValue = PainlessLookupUtility.typeToJavaType(rtn);
-        } else if (Modifier.isStatic(modifiers)) {
-            // static method: straightforward copy
-            params = new Class<?>[arguments.size()];
-            for (int i = 0; i < arguments.size(); i++) {
-                params[i] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
-            }
-            returnValue = PainlessLookupUtility.typeToJavaType(rtn);
-        } else if ("<init>".equals(name)) {
-            // constructor: returns the owner class
-            params = new Class<?>[arguments.size()];
-            for (int i = 0; i < arguments.size(); i++) {
-                params[i] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
-            }
-            returnValue = target;
-        } else {
-            // virtual/interface method: add receiver class
-            params = new Class<?>[1 + arguments.size()];
-            params[0] = target;
-            for (int i = 0; i < arguments.size(); i++) {
-                params[i + 1] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
-            }
-            returnValue = PainlessLookupUtility.typeToJavaType(rtn);
-        }
-        return MethodType.methodType(returnValue, params);
-    }
-
-    public void write(MethodWriter writer) {
-        final org.objectweb.asm.Type type;
-        final Class<?> clazz;
-        if (augmentation != null) {
-            assert Modifier.isStatic(modifiers);
-            clazz = augmentation;
-            type = org.objectweb.asm.Type.getType(augmentation);
-        } else {
-            clazz = target;
-            type = Type.getType(target);
-        }
-
-        if (Modifier.isStatic(modifiers)) {
-            // invokeStatic assumes that the owner class is not an interface, so this is a
-            // special case for interfaces where the interface method boolean needs to be set to
-            // true to reference the appropriate class constant when calling a static interface
-            // method since java 8 did not check, but java 9 and 10 do
-            if (Modifier.isInterface(clazz.getModifiers())) {
-                writer.visitMethodInsn(Opcodes.INVOKESTATIC,
-                        type.getInternalName(), name, getMethodType().toMethodDescriptorString(), true);
-            } else {
-                writer.invokeStatic(type, method);
-            }
-        } else if (Modifier.isInterface(clazz.getModifiers())) {
-            writer.invokeInterface(type, method);
-        } else {
-            writer.invokeVirtual(type, method);
-        }
+    public final Method javaMethod;
+    public final Class<?> targetClass;
+    public final Class<?> returnType;
+    public final List<Class<?>> typeParameters;
+    public final MethodHandle methodHandle;
+    public final MethodType methodType;
+
+    public PainlessMethod(Method javaMethod, Class<?> targetClass, Class<?> returnType, List<Class<?>> typeParameters,
+            MethodHandle methodHandle, MethodType methodType) {
+
+        this.javaMethod = javaMethod;
+        this.targetClass = targetClass;
+        this.returnType = returnType;
+        this.typeParameters = Collections.unmodifiableList(typeParameters);
+        this.methodHandle = methodHandle;
+        this.methodType = methodType;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
index 098c75386e1a6..1f9973df19224 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
@@ -21,10 +21,10 @@
 
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.LocalMethod;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.objectweb.asm.commons.Method;
 
 import java.util.List;
 import java.util.Objects;
@@ -40,7 +40,7 @@ public final class ECallLocal extends AExpression {
     private final String name;
     private final List<AExpression> arguments;
 
-    private PainlessMethod method = null;
+    private LocalMethod method = null;
 
     public ECallLocal(Location location, String name, List<AExpression> arguments) {
         super(location);
@@ -58,8 +58,7 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-        String methodKey = PainlessLookupUtility.buildPainlessMethodKey(name, arguments.size());
-        method = locals.getMethod(methodKey);
+        method = locals.getMethod(name, arguments.size());
 
         if (method == null) {
             throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() + "] arguments."));
@@ -68,14 +67,14 @@ void analyze(Locals locals) {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            expression.expected = method.arguments.get(argument);
+            expression.expected = method.typeParameters.get(argument);
             expression.internal = true;
             expression.analyze(locals);
             arguments.set(argument, expression.cast(locals));
         }
 
         statement = true;
-        actual = method.rtn;
+        actual = method.returnType;
     }
 
     @Override
@@ -86,7 +85,7 @@ void write(MethodWriter writer, Globals globals) {
             argument.write(writer, globals);
         }
 
-        writer.invokeStatic(CLASS_TYPE, method.method);
+        writer.invokeStatic(CLASS_TYPE, new Method(method.name, method.methodType.toMethodDescriptorString()));
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index 7b35bc1b48ee5..a649fa7611c65 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Globals;
@@ -35,8 +34,6 @@
 import java.util.Objects;
 import java.util.Set;
 
-import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
-
 /**
  * Represents a capturing function reference.
  */
@@ -76,23 +73,8 @@ void analyze(Locals locals) {
             defPointer = null;
             // static case
             if (captured.clazz != def.class) {
-                try {
-                    ref = new FunctionRef(locals.getPainlessLookup(), expected,
-                            PainlessLookupUtility.typeToCanonicalTypeName(captured.clazz), call, 1);
-
-                    // check casts between the interface method and the delegate method are legal
-                    for (int i = 0; i < ref.interfaceMethod.arguments.size(); ++i) {
-                        Class<?> from = ref.interfaceMethod.arguments.get(i);
-                        Class<?> to = ref.delegateMethod.arguments.get(i);
-                        AnalyzerCaster.getLegalCast(location, from, to, false, true);
-                    }
-
-                    if (ref.interfaceMethod.rtn != void.class) {
-                        AnalyzerCaster.getLegalCast(location, ref.delegateMethod.rtn, ref.interfaceMethod.rtn, false, true);
-                    }
-                } catch (IllegalArgumentException e) {
-                    throw createError(e);
-                }
+                ref = FunctionRef.create(locals.getPainlessLookup(), locals.getMethods(), location,
+                        expected, PainlessLookupUtility.typeToCanonicalTypeName(captured.clazz), call, 1);
             }
             actual = expected;
         }
@@ -114,17 +96,7 @@ void write(MethodWriter writer, Globals globals) {
         } else {
             // typed interface, typed implementation
             writer.visitVarInsn(MethodWriter.getType(captured.clazz).getOpcode(Opcodes.ILOAD), captured.getSlot());
-            writer.invokeDynamic(
-                ref.interfaceMethodName,
-                ref.factoryDescriptor,
-                LAMBDA_BOOTSTRAP_HANDLE,
-                ref.interfaceType,
-                ref.delegateClassName,
-                ref.delegateInvokeType,
-                ref.delegateMethodName,
-                ref.delegateType,
-                ref.isDelegateInterface ? 1 : 0
-            );
+            writer.invokeLambdaCall(ref);
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index b07613714b8ef..08236a965fe52 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -63,6 +63,6 @@ void write(MethodWriter writer, Globals globals) {
 
     @Override
     public String toString() {
-        return singleLineToString(PainlessLookupUtility.typeToCanonicalTypeName(cast.to), child);
+        return singleLineToString(PainlessLookupUtility.typeToCanonicalTypeName(cast.targetType), child);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index d19068f8fa6a2..3ad3018c61e34 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -49,9 +49,9 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-        try {
-            actual = locals.getPainlessLookup().getJavaClassFromPainlessType(type);
-        } catch (IllegalArgumentException exception) {
+        actual = locals.getPainlessLookup().canonicalTypeNameToType(type);
+
+        if (actual == null) {
             throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index d787db5d41c92..c97cc66c7c7ca 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -19,21 +19,16 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.objectweb.asm.Type;
 
 import java.util.Objects;
 import java.util.Set;
 
-import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
-
 /**
  * Represents a function reference.
  */
@@ -62,40 +57,7 @@ void analyze(Locals locals) {
             defPointer = "S" + type + "." + call + ",0";
         } else {
             defPointer = null;
-            try {
-                if ("this".equals(type)) {
-                    // user's own function
-                    PainlessMethod interfaceMethod = locals.getPainlessLookup().getPainlessStructFromJavaClass(expected).functionalMethod;
-                    if (interfaceMethod == null) {
-                        throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                                "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface");
-                    }
-                    PainlessMethod delegateMethod =
-                            locals.getMethod(PainlessLookupUtility.buildPainlessMethodKey(call, interfaceMethod.arguments.size()));
-                    if (delegateMethod == null) {
-                        throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                                "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], function not found");
-                    }
-                    ref = new FunctionRef(expected, interfaceMethod, delegateMethod, 0);
-
-                    // check casts between the interface method and the delegate method are legal
-                    for (int i = 0; i < interfaceMethod.arguments.size(); ++i) {
-                        Class<?> from = interfaceMethod.arguments.get(i);
-                        Class<?> to = delegateMethod.arguments.get(i);
-                        AnalyzerCaster.getLegalCast(location, from, to, false, true);
-                    }
-
-                    if (interfaceMethod.rtn != void.class) {
-                        AnalyzerCaster.getLegalCast(location, delegateMethod.rtn, interfaceMethod.rtn, false, true);
-                    }
-                } else {
-                    // whitelist lookup
-                    ref = new FunctionRef(locals.getPainlessLookup(), expected, type, call, 0);
-                }
-
-            } catch (IllegalArgumentException e) {
-                throw createError(e);
-            }
+            ref = FunctionRef.create(locals.getPainlessLookup(), locals.getMethods(), location, expected, type, call, 0);
             actual = expected;
         }
     }
@@ -104,17 +66,7 @@ void analyze(Locals locals) {
     void write(MethodWriter writer, Globals globals) {
         if (ref != null) {
             writer.writeDebugInfo(location);
-            writer.invokeDynamic(
-                ref.interfaceMethodName,
-                ref.factoryDescriptor,
-                LAMBDA_BOOTSTRAP_HANDLE,
-                ref.interfaceType,
-                ref.delegateClassName,
-                ref.delegateInvokeType,
-                ref.delegateMethodName,
-                ref.delegateType,
-                ref.isDelegateInterface ? 1 : 0
-            );
+            writer.invokeLambdaCall(ref);
         } else {
             // TODO: don't do this: its just to cutover :)
             writer.push((String)null);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
index 2fa8ca8ca9513..73e4f176ea1ba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
@@ -54,12 +54,11 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-        Class<?> clazz;
 
         // ensure the specified type is part of the definition
-        try {
-            clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type);
-        } catch (IllegalArgumentException exception) {
+        Class<?> clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
+
+        if (clazz == null) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index ab1442be805eb..af906416ca7bc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
@@ -39,8 +38,6 @@
 import java.util.Objects;
 import java.util.Set;
 
-import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
-
 /**
  * Lambda expression node.
  * <p>
@@ -118,29 +115,30 @@ void analyze(Locals locals) {
                     actualParamTypeStrs.add(type);
                 }
             }
+
         } else {
             // we know the method statically, infer return type and any unknown/def types
-            interfaceMethod = locals.getPainlessLookup().getPainlessStructFromJavaClass(expected).functionalMethod;
+            interfaceMethod = locals.getPainlessLookup().lookupFunctionalInterfacePainlessMethod(expected);
             if (interfaceMethod == null) {
                 throw createError(new IllegalArgumentException("Cannot pass lambda to " +
                         "[" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface"));
             }
             // check arity before we manipulate parameters
-            if (interfaceMethod.arguments.size() != paramTypeStrs.size())
-                throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name +
+            if (interfaceMethod.typeParameters.size() != paramTypeStrs.size())
+                throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.javaMethod.getName() +
                         "] in [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "]");
             // for method invocation, its allowed to ignore the return value
-            if (interfaceMethod.rtn == void.class) {
+            if (interfaceMethod.returnType == void.class) {
                 returnType = def.class;
             } else {
-                returnType = interfaceMethod.rtn;
+                returnType = interfaceMethod.returnType;
             }
             // replace any null types with the actual type
             actualParamTypeStrs = new ArrayList<>(paramTypeStrs.size());
             for (int i = 0; i < paramTypeStrs.size(); i++) {
                 String paramType = paramTypeStrs.get(i);
                 if (paramType == null) {
-                    actualParamTypeStrs.add(PainlessLookupUtility.typeToCanonicalTypeName(interfaceMethod.arguments.get(i)));
+                    actualParamTypeStrs.add(PainlessLookupUtility.typeToCanonicalTypeName(interfaceMethod.typeParameters.get(i)));
                 } else {
                     actualParamTypeStrs.add(paramType);
                 }
@@ -172,7 +170,7 @@ void analyze(Locals locals) {
         desugared = new SFunction(reserved, location, PainlessLookupUtility.typeToCanonicalTypeName(returnType), name,
                                   paramTypes, paramNames, statements, true);
         desugared.generateSignature(locals.getPainlessLookup());
-        desugared.analyze(Locals.newLambdaScope(locals.getProgramScope(), returnType,
+        desugared.analyze(Locals.newLambdaScope(locals.getProgramScope(), desugared.name, returnType,
                                                 desugared.parameters, captures.size(), reserved.getMaxLoopCounter()));
 
         // setup method reference to synthetic method
@@ -182,23 +180,8 @@ void analyze(Locals locals) {
             defPointer = "Sthis." + name + "," + captures.size();
         } else {
             defPointer = null;
-            try {
-                ref = new FunctionRef(expected, interfaceMethod, desugared.method, captures.size());
-            } catch (IllegalArgumentException e) {
-                throw createError(e);
-            }
-
-            // check casts between the interface method and the delegate method are legal
-            for (int i = 0; i < interfaceMethod.arguments.size(); ++i) {
-                Class<?> from = interfaceMethod.arguments.get(i);
-                Class<?> to = desugared.parameters.get(i + captures.size()).clazz;
-                AnalyzerCaster.getLegalCast(location, from, to, false, true);
-            }
-
-            if (interfaceMethod.rtn != void.class) {
-                AnalyzerCaster.getLegalCast(location, desugared.rtnType, interfaceMethod.rtn, false, true);
-            }
-
+            ref = FunctionRef.create(
+                    locals.getPainlessLookup(), locals.getMethods(), location, expected, "this", desugared.name, captures.size());
             actual = expected;
         }
     }
@@ -214,17 +197,7 @@ void write(MethodWriter writer, Globals globals) {
                 writer.visitVarInsn(MethodWriter.getType(capture.clazz).getOpcode(Opcodes.ILOAD), capture.getSlot());
             }
 
-            writer.invokeDynamic(
-                ref.interfaceMethodName,
-                ref.factoryDescriptor,
-                LAMBDA_BOOTSTRAP_HANDLE,
-                ref.interfaceType,
-                ref.delegateClassName,
-                ref.delegateInvokeType,
-                ref.delegateMethodName,
-                ref.delegateType,
-                ref.isDelegateInterface ? 1 : 0
-            );
+            writer.invokeLambdaCall(ref);
         } else {
             // placeholder
             writer.push((String)null);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index e0af653d2098a..8c9154aaaf304 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -23,22 +23,25 @@
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Type;
+import org.objectweb.asm.commons.Method;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
+
 /**
  * Represents a list initialization shortcut.
  */
 public final class EListInit extends AExpression {
     private final List<AExpression> values;
 
-    private PainlessMethod constructor = null;
+    private PainlessConstructor constructor = null;
     private PainlessMethod method = null;
 
     public EListInit(Location location, List<AExpression> values) {
@@ -62,18 +65,17 @@ void analyze(Locals locals) {
 
         actual = ArrayList.class;
 
-        constructor = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors
-                .get(PainlessLookupUtility.buildPainlessMethodKey("<init>", 0));
+        constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, 0);
 
         if (constructor == null) {
-            throw createError(new IllegalStateException("Illegal tree structure."));
+            throw createError(new IllegalArgumentException(
+                    "constructor [" + typeToCanonicalTypeName(actual) + ", <init>/0] not found"));
         }
 
-        method = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).methods
-                .get(PainlessLookupUtility.buildPainlessMethodKey("add", 1));
+        method = locals.getPainlessLookup().lookupPainlessMethod(actual, false, "add", 1);
 
         if (method == null) {
-            throw createError(new IllegalStateException("Illegal tree structure."));
+            throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(actual) + ", add/1] not found"));
         }
 
         for (int index = 0; index < values.size(); ++index) {
@@ -92,12 +94,13 @@ void write(MethodWriter writer, Globals globals) {
 
         writer.newInstance(MethodWriter.getType(actual));
         writer.dup();
-        writer.invokeConstructor(Type.getType(constructor.target), constructor.method);
+        writer.invokeConstructor(
+                    Type.getType(constructor.javaConstructor.getDeclaringClass()), Method.getMethod(constructor.javaConstructor));
 
         for (AExpression value : values) {
             writer.dup();
             value.write(writer, globals);
-            method.write(writer);
+            writer.invokeMethodCall(method);
             writer.pop();
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index d81f08dc3cc54..11c12b2cd0a96 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -23,15 +23,18 @@
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Type;
+import org.objectweb.asm.commons.Method;
 
 import java.util.HashMap;
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
+
 /**
  * Represents a map initialization shortcut.
  */
@@ -39,7 +42,7 @@ public final class EMapInit extends AExpression {
     private final List<AExpression> keys;
     private final List<AExpression> values;
 
-    private PainlessMethod constructor = null;
+    private PainlessConstructor constructor = null;
     private PainlessMethod method = null;
 
     public EMapInit(Location location, List<AExpression> keys, List<AExpression> values) {
@@ -68,18 +71,17 @@ void analyze(Locals locals) {
 
         actual = HashMap.class;
 
-        constructor = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors
-                .get(PainlessLookupUtility.buildPainlessMethodKey("<init>", 0));
+        constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, 0);
 
         if (constructor == null) {
-            throw createError(new IllegalStateException("Illegal tree structure."));
+            throw createError(new IllegalArgumentException(
+                    "constructor [" + typeToCanonicalTypeName(actual) + ", <init>/0] not found"));
         }
 
-        method = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).methods
-                .get(PainlessLookupUtility.buildPainlessMethodKey("put", 2));
+        method = locals.getPainlessLookup().lookupPainlessMethod(actual, false, "put", 2);
 
         if (method == null) {
-            throw createError(new IllegalStateException("Illegal tree structure."));
+            throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(actual) + ", put/2] not found"));
         }
 
         if (keys.size() != values.size()) {
@@ -111,7 +113,8 @@ void write(MethodWriter writer, Globals globals) {
 
         writer.newInstance(MethodWriter.getType(actual));
         writer.dup();
-        writer.invokeConstructor(Type.getType(constructor.target), constructor.method);
+        writer.invokeConstructor(
+                    Type.getType(constructor.javaConstructor.getDeclaringClass()), Method.getMethod(constructor.javaConstructor));
 
         for (int index = 0; index < keys.size(); ++index) {
             AExpression key = keys.get(index);
@@ -120,7 +123,7 @@ void write(MethodWriter writer, Globals globals) {
             writer.dup();
             key.write(writer, globals);
             value.write(writer, globals);
-            method.write(writer);
+            writer.invokeMethodCall(method);
             writer.pop();
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
index f9bd4cebc3fed..cef005de9c3bc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
@@ -54,15 +54,13 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-         if (!read) {
-            throw createError(new IllegalArgumentException("A newly created array must be read from."));
+        if (!read) {
+             throw createError(new IllegalArgumentException("A newly created array must be read from."));
         }
 
-        Class<?> clazz;
+        Class<?> clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
 
-        try {
-            clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type);
-        } catch (IllegalArgumentException exception) {
+        if (clazz == null) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
index f092a17c9fc4a..9423ed5d109de 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
@@ -23,15 +23,17 @@
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.objectweb.asm.Type;
+import org.objectweb.asm.commons.Method;
 
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
+
 /**
  * Represents and object instantiation.
  */
@@ -40,7 +42,7 @@ public final class ENewObj extends AExpression {
     private final String type;
     private final List<AExpression> arguments;
 
-    private PainlessMethod constructor;
+    private PainlessConstructor constructor;
 
     public ENewObj(Location location, String type, List<AExpression> arguments) {
         super(location);
@@ -58,39 +60,38 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-        try {
-            actual = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type);
-        } catch (IllegalArgumentException exception) {
+        actual = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
+
+        if (actual == null) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
-        PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual);
-        constructor = struct.constructors.get(PainlessLookupUtility.buildPainlessMethodKey("<init>", arguments.size()));
+        constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, arguments.size());
 
-        if (constructor != null) {
-            Class<?>[] types = new Class<?>[constructor.arguments.size()];
-            constructor.arguments.toArray(types);
+        if (constructor == null) {
+            throw createError(new IllegalArgumentException(
+                    "constructor [" + typeToCanonicalTypeName(actual) + ", <init>/" + arguments.size() + "] not found"));
+        }
 
-            if (constructor.arguments.size() != arguments.size()) {
-                throw createError(new IllegalArgumentException(
-                        "When calling constructor on type [" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "] " +
-                        "expected [" + constructor.arguments.size() + "] arguments, but found [" + arguments.size() + "]."));
-            }
+        Class<?>[] types = new Class<?>[constructor.typeParameters.size()];
+        constructor.typeParameters.toArray(types);
 
-            for (int argument = 0; argument < arguments.size(); ++argument) {
-                AExpression expression = arguments.get(argument);
+        if (constructor.typeParameters.size() != arguments.size()) {
+            throw createError(new IllegalArgumentException(
+                    "When calling constructor on type [" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "] " +
+                    "expected [" + constructor.typeParameters.size() + "] arguments, but found [" + arguments.size() + "]."));
+        }
 
-                expression.expected = types[argument];
-                expression.internal = true;
-                expression.analyze(locals);
-                arguments.set(argument, expression.cast(locals));
-            }
+        for (int argument = 0; argument < arguments.size(); ++argument) {
+            AExpression expression = arguments.get(argument);
 
-            statement = true;
-        } else {
-            throw createError(new IllegalArgumentException(
-                    "Unknown new call on type [" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "]."));
+            expression.expected = types[argument];
+            expression.internal = true;
+            expression.analyze(locals);
+            arguments.set(argument, expression.cast(locals));
         }
+
+        statement = true;
     }
 
     @Override
@@ -107,7 +108,8 @@ void write(MethodWriter writer, Globals globals) {
             argument.write(writer, globals);
         }
 
-        writer.invokeConstructor(Type.getType(constructor.target), constructor.method);
+        writer.invokeConstructor(
+                    Type.getType(constructor.javaConstructor.getDeclaringClass()), Method.getMethod(constructor.javaConstructor));
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
index a556b3ad315c6..0d8c94db0f1fc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
@@ -47,9 +47,9 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-        try {
-            actual = locals.getPainlessLookup().getJavaClassFromPainlessType(type);
-        } catch (IllegalArgumentException exception) {
+        actual = locals.getPainlessLookup().canonicalTypeNameToType(type);
+
+        if (actual == null) {
             throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
index 56bc18eadbd61..25ae1ed97742a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
@@ -23,8 +23,6 @@
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.lookup.PainlessClass;
-import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.def;
 
@@ -32,6 +30,8 @@
 import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
+
 /**
  * Represents a method call and defers to a child subnode.
  */
@@ -66,26 +66,18 @@ void analyze(Locals locals) {
         prefix.expected = prefix.actual;
         prefix = prefix.cast(locals);
 
-        if (prefix.actual.isArray()) {
-            throw createError(new IllegalArgumentException("Illegal call [" + name + "] on array type."));
-        }
-
-        PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual);
-
-        if (prefix.actual.isPrimitive()) {
-            struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(PainlessLookupUtility.typeToBoxedType(prefix.actual));
-        }
+        if (prefix.actual == def.class) {
+            sub = new PSubDefCall(location, name, arguments);
+        } else {
+            PainlessMethod method =
+                    locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, prefix instanceof EStatic, name, arguments.size());
 
-        String methodKey = PainlessLookupUtility.buildPainlessMethodKey(name, arguments.size());
-        PainlessMethod method = prefix instanceof EStatic ? struct.staticMethods.get(methodKey) : struct.methods.get(methodKey);
+            if (method == null) {
+                throw createError(new IllegalArgumentException(
+                        "method [" + typeToCanonicalTypeName(prefix.actual) + ", " + name + "/" + arguments.size() + "] not found"));
+            }
 
-        if (method != null) {
             sub = new PSubCallInvoke(location, method, prefix.actual, arguments);
-        } else if (prefix.actual == def.class) {
-            sub = new PSubDefCall(location, name, arguments);
-        } else {
-            throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() + "] arguments " +
-                    "on type [" + PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual) + "]."));
         }
 
         if (nullSafe) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
index b322d5b1f287f..7efd6a29899c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessField;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
@@ -34,6 +33,8 @@
 import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
+
 /**
  * Represents a field load/store and defers to a child subnode.
  */
@@ -67,26 +68,25 @@ void analyze(Locals locals) {
         } else if (prefix.actual == def.class) {
             sub = new PSubDefField(location, value);
         } else {
-            PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual);
-            PainlessField field = prefix instanceof EStatic ? struct.staticFields.get(value) : struct.fields.get(value);
+            PainlessField field = locals.getPainlessLookup().lookupPainlessField(prefix.actual, prefix instanceof EStatic, value);
 
-            if (field != null) {
-                sub = new PSubField(location, field);
-            } else {
-                PainlessMethod getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(
-                        "get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
+            if (field == null) {
+                PainlessMethod getter;
+                PainlessMethod setter;
+
+                getter = locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
+                        "get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
 
                 if (getter == null) {
-                    getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(
-                            "is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
+                    getter = locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
+                            "is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
                 }
 
-                PainlessMethod setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(
-                        "set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
+                setter = locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
+                        "set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
 
                 if (getter != null || setter != null) {
-                    sub = new PSubShortcut(
-                            location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual), getter, setter);
+                    sub = new PSubShortcut(location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual), getter, setter);
                 } else {
                     EConstant index = new EConstant(location, value);
                     index.analyze(locals);
@@ -99,12 +99,14 @@ void analyze(Locals locals) {
                         sub = new PSubListShortcut(location, prefix.actual, index);
                     }
                 }
-            }
-        }
 
-        if (sub == null) {
-            throw createError(new IllegalArgumentException(
-                "Unknown field [" + value + "] for type [" + PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual) + "]."));
+                if (sub == null) {
+                    throw createError(new IllegalArgumentException(
+                            "field [" + typeToCanonicalTypeName(prefix.actual) + ", " + value + "] not found"));
+                }
+            } else {
+                sub = new PSubField(location, field);
+            }
         }
 
         if (nullSafe) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
index 237efa61ffa7d..fe2ae52603b57 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
@@ -56,14 +56,14 @@ void analyze(Locals locals) {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            expression.expected = method.arguments.get(argument);
+            expression.expected = method.typeParameters.get(argument);
             expression.internal = true;
             expression.analyze(locals);
             arguments.set(argument, expression.cast(locals));
         }
 
         statement = true;
-        actual = method.rtn;
+        actual = method.returnType;
     }
 
     @Override
@@ -78,11 +78,11 @@ void write(MethodWriter writer, Globals globals) {
             argument.write(writer, globals);
         }
 
-        method.write(writer);
+        writer.invokeMethodCall(method);
     }
 
     @Override
     public String toString() {
-        return singleLineToStringWithOptionalArgs(arguments, prefix, method.name);
+        return singleLineToStringWithOptionalArgs(arguments, prefix, method.javaMethod.getName());
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
index 007a599e9f842..9e09f810250f0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
@@ -51,22 +51,24 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-         if (write && Modifier.isFinal(field.modifiers)) {
-             throw createError(new IllegalArgumentException("Cannot write to read-only field [" + field.name + "] for type " +
-                     "[" + PainlessLookupUtility.typeToCanonicalTypeName(field.clazz) + "]."));
+         if (write && Modifier.isFinal(field.javaField.getModifiers())) {
+             throw createError(new IllegalArgumentException("Cannot write to read-only field [" + field.javaField.getName() + "] " +
+                     "for type [" + PainlessLookupUtility.typeToCanonicalTypeName(field.javaField.getDeclaringClass()) + "]."));
          }
 
-        actual = field.clazz;
+        actual = field.typeParameter;
     }
 
     @Override
     void write(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
-            writer.getStatic(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
+        if (java.lang.reflect.Modifier.isStatic(field.javaField.getModifiers())) {
+            writer.getStatic(Type.getType(
+                    field.javaField.getDeclaringClass()), field.javaField.getName(), MethodWriter.getType(field.typeParameter));
         } else {
-            writer.getField(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
+            writer.getField(Type.getType(
+                    field.javaField.getDeclaringClass()), field.javaField.getName(), MethodWriter.getType(field.typeParameter));
         }
     }
 
@@ -94,10 +96,12 @@ void setup(MethodWriter writer, Globals globals) {
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
-            writer.getStatic(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
+        if (java.lang.reflect.Modifier.isStatic(field.javaField.getModifiers())) {
+            writer.getStatic(Type.getType(
+                    field.javaField.getDeclaringClass()), field.javaField.getName(), MethodWriter.getType(field.typeParameter));
         } else {
-            writer.getField(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
+            writer.getField(Type.getType(
+                    field.javaField.getDeclaringClass()), field.javaField.getName(), MethodWriter.getType(field.typeParameter));
         }
     }
 
@@ -105,15 +109,17 @@ void load(MethodWriter writer, Globals globals) {
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
-            writer.putStatic(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
+        if (java.lang.reflect.Modifier.isStatic(field.javaField.getModifiers())) {
+            writer.putStatic(Type.getType(
+                    field.javaField.getDeclaringClass()), field.javaField.getName(), MethodWriter.getType(field.typeParameter));
         } else {
-            writer.putField(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
+            writer.putField(Type.getType(
+                    field.javaField.getDeclaringClass()), field.javaField.getName(), MethodWriter.getType(field.typeParameter));
         }
     }
 
     @Override
     public String toString() {
-        return singleLineToString(prefix, field.name);
+        return singleLineToString(prefix, field.javaField.getName());
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
index 509aad6415349..3bc4913fde940 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.WriterConstants;
-import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 
@@ -56,23 +55,22 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-        PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(targetClass);
         String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
 
-        getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("get", 1));
-        setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("set", 2));
+        getter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
+        setter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "set", 2);
 
-        if (getter != null && (getter.rtn == void.class || getter.arguments.size() != 1 ||
-            getter.arguments.get(0) != int.class)) {
+        if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1 ||
+            getter.typeParameters.get(0) != int.class)) {
             throw createError(new IllegalArgumentException("Illegal list get shortcut for type [" + canonicalClassName + "]."));
         }
 
-        if (setter != null && (setter.arguments.size() != 2 || setter.arguments.get(0) != int.class)) {
+        if (setter != null && (setter.typeParameters.size() != 2 || setter.typeParameters.get(0) != int.class)) {
             throw createError(new IllegalArgumentException("Illegal list set shortcut for type [" + canonicalClassName + "]."));
         }
 
-        if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0))
-            || !getter.rtn.equals(setter.arguments.get(1)))) {
+        if (getter != null && setter != null && (!getter.typeParameters.get(0).equals(setter.typeParameters.get(0))
+            || !getter.returnType.equals(setter.typeParameters.get(1)))) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
@@ -81,7 +79,7 @@ void analyze(Locals locals) {
             index.analyze(locals);
             index = index.cast(locals);
 
-            actual = setter != null ? setter.arguments.get(1) : getter.rtn;
+            actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal list shortcut for type [" + canonicalClassName + "]."));
         }
@@ -119,21 +117,18 @@ void setup(MethodWriter writer, Globals globals) {
     @Override
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
+        writer.invokeMethodCall(getter);
 
-        getter.write(writer);
-
-        if (getter.rtn == getter.handle.type().returnType()) {
-            writer.checkCast(MethodWriter.getType(getter.rtn));
+        if (getter.returnType == getter.javaMethod.getReturnType()) {
+            writer.checkCast(MethodWriter.getType(getter.returnType));
         }
     }
 
     @Override
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
-
-        setter.write(writer);
-
-        writer.writePop(MethodWriter.getType(setter.rtn).getSize());
+        writer.invokeMethodCall(setter);
+        writer.writePop(MethodWriter.getType(setter.returnType).getSize());
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
index 2d7f2250c6c38..0a0f099bd6841 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 
@@ -55,31 +54,30 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-        PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(targetClass);
         String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
 
-        getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("get", 1));
-        setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("put", 2));
+        getter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
+        setter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "put", 2);
 
-        if (getter != null && (getter.rtn == void.class || getter.arguments.size() != 1)) {
+        if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1)) {
             throw createError(new IllegalArgumentException("Illegal map get shortcut for type [" + canonicalClassName + "]."));
         }
 
-        if (setter != null && setter.arguments.size() != 2) {
+        if (setter != null && setter.typeParameters.size() != 2) {
             throw createError(new IllegalArgumentException("Illegal map set shortcut for type [" + canonicalClassName + "]."));
         }
 
-        if (getter != null && setter != null &&
-            (!getter.arguments.get(0).equals(setter.arguments.get(0)) || !getter.rtn.equals(setter.arguments.get(1)))) {
+        if (getter != null && setter != null && (!getter.typeParameters.get(0).equals(setter.typeParameters.get(0)) ||
+                !getter.returnType.equals(setter.typeParameters.get(1)))) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
         if ((read || write) && (!read || getter != null) && (!write || setter != null)) {
-            index.expected = setter != null ? setter.arguments.get(0) : getter.arguments.get(0);
+            index.expected = setter != null ? setter.typeParameters.get(0) : getter.typeParameters.get(0);
             index.analyze(locals);
             index = index.cast(locals);
 
-            actual = setter != null ? setter.arguments.get(1) : getter.rtn;
+            actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal map shortcut for type [" + canonicalClassName + "]."));
         }
@@ -90,11 +88,10 @@ void write(MethodWriter writer, Globals globals) {
         index.write(writer, globals);
 
         writer.writeDebugInfo(location);
+        writer.invokeMethodCall(getter);
 
-        getter.write(writer);
-
-        if (getter.rtn != getter.handle.type().returnType()) {
-            writer.checkCast(MethodWriter.getType(getter.rtn));
+        if (getter.returnType != getter.javaMethod.getReturnType()) {
+            writer.checkCast(MethodWriter.getType(getter.returnType));
         }
     }
 
@@ -121,21 +118,18 @@ void setup(MethodWriter writer, Globals globals) {
     @Override
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
+        writer.invokeMethodCall(getter);
 
-        getter.write(writer);
-
-        if (getter.rtn != getter.handle.type().returnType()) {
-            writer.checkCast(MethodWriter.getType(getter.rtn));
+        if (getter.returnType != getter.javaMethod.getReturnType()) {
+            writer.checkCast(MethodWriter.getType(getter.returnType));
         }
     }
 
     @Override
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
-
-        setter.write(writer);
-
-        writer.writePop(MethodWriter.getType(setter.rtn).getSize());
+        writer.invokeMethodCall(setter);
+        writer.writePop(MethodWriter.getType(setter.returnType).getSize());
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
index eb5668c554c20..1697566047798 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
@@ -53,22 +53,22 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-        if (getter != null && (getter.rtn == void.class || !getter.arguments.isEmpty())) {
+        if (getter != null && (getter.returnType == void.class || !getter.typeParameters.isEmpty())) {
             throw createError(new IllegalArgumentException(
                 "Illegal get shortcut on field [" + value + "] for type [" + type + "]."));
         }
 
-        if (setter != null && (setter.rtn != void.class || setter.arguments.size() != 1)) {
+        if (setter != null && (setter.returnType != void.class || setter.typeParameters.size() != 1)) {
             throw createError(new IllegalArgumentException(
                 "Illegal set shortcut on field [" + value + "] for type [" + type + "]."));
         }
 
-        if (getter != null && setter != null && setter.arguments.get(0) != getter.rtn) {
+        if (getter != null && setter != null && setter.typeParameters.get(0) != getter.returnType) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
         if ((getter != null || setter != null) && (!read || getter != null) && (!write || setter != null)) {
-            actual = setter != null ? setter.arguments.get(0) : getter.rtn;
+            actual = setter != null ? setter.typeParameters.get(0) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal shortcut on field [" + value + "] for type [" + type + "]."));
         }
@@ -78,10 +78,10 @@ void analyze(Locals locals) {
     void write(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        getter.write(writer);
+        writer.invokeMethodCall(getter);
 
-        if (!getter.rtn.equals(getter.handle.type().returnType())) {
-            writer.checkCast(MethodWriter.getType(getter.rtn));
+        if (!getter.returnType.equals(getter.javaMethod.getReturnType())) {
+            writer.checkCast(MethodWriter.getType(getter.returnType));
         }
     }
 
@@ -109,10 +109,10 @@ void setup(MethodWriter writer, Globals globals) {
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        getter.write(writer);
+        writer.invokeMethodCall(getter);
 
-        if (getter.rtn != getter.handle.type().returnType()) {
-            writer.checkCast(MethodWriter.getType(getter.rtn));
+        if (getter.returnType != getter.javaMethod.getReturnType()) {
+            writer.checkCast(MethodWriter.getType(getter.returnType));
         }
     }
 
@@ -120,9 +120,9 @@ void load(MethodWriter writer, Globals globals) {
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        setter.write(writer);
+        writer.invokeMethodCall(setter);
 
-        writer.writePop(MethodWriter.getType(setter.rtn).getSize());
+        writer.writePop(MethodWriter.getType(setter.returnType).getSize());
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index 8a703c80cba2f..0c8ba5de6b2cf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -64,11 +64,9 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-        Class<?> clazz;
+        Class<?> clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
 
-        try {
-            clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type);
-        } catch (IllegalArgumentException exception) {
+        if (clazz == null) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index fb92c20e89e01..7ead673c70b7a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -59,11 +59,9 @@ void extractVariables(Set<String> variables) {
 
     @Override
     void analyze(Locals locals) {
-        Class<?> clazz;
+        Class<?> clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
 
-        try {
-            clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type);
-        } catch (IllegalArgumentException exception) {
+        if (clazz == null) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 9ff57e6b913cc..cf41105c4fe36 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -68,11 +68,9 @@ void analyze(Locals locals) {
         expression.expected = expression.actual;
         expression = expression.cast(locals);
 
-        Class<?> clazz;
+        Class<?> clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
 
-        try {
-            clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type);
-        } catch (IllegalArgumentException exception) {
+        if (clazz == null) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 7c243e296c7e3..6fe09627f9dfd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -20,25 +20,19 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Constant;
-import org.elasticsearch.painless.Def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Parameter;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.WriterConstants;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.node.SSource.Reserved;
 import org.objectweb.asm.ClassVisitor;
-import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
 
 import java.lang.invoke.MethodType;
-import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -48,7 +42,6 @@
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.unmodifiableSet;
-import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
 
 /**
  * Represents a user-defined function.
@@ -92,9 +85,12 @@ public int getMaxLoopCounter() {
     private final List<AStatement> statements;
     public final boolean synthetic;
 
-    Class<?> rtnType = null;
+    Class<?> returnType;
+    List<Class<?>> typeParameters;
+    MethodType methodType;
+
+    org.objectweb.asm.commons.Method method;
     List<Parameter> parameters = new ArrayList<>();
-    PainlessMethod method = null;
 
     private Variable loop = null;
 
@@ -119,9 +115,9 @@ void extractVariables(Set<String> variables) {
     }
 
     void generateSignature(PainlessLookup painlessLookup) {
-        try {
-            rtnType = painlessLookup.getJavaClassFromPainlessType(rtnTypeStr);
-        } catch (IllegalArgumentException exception) {
+        returnType = painlessLookup.canonicalTypeNameToType(rtnTypeStr);
+
+        if (returnType == null) {
             throw createError(new IllegalArgumentException("Illegal return type [" + rtnTypeStr + "] for function [" + name + "]."));
         }
 
@@ -133,21 +129,22 @@ void generateSignature(PainlessLookup painlessLookup) {
         List<Class<?>> paramTypes = new ArrayList<>();
 
         for (int param = 0; param < this.paramTypeStrs.size(); ++param) {
-            try {
-                Class<?> paramType = painlessLookup.getJavaClassFromPainlessType(this.paramTypeStrs.get(param));
+                Class<?> paramType = painlessLookup.canonicalTypeNameToType(this.paramTypeStrs.get(param));
 
-                paramClasses[param] = PainlessLookupUtility.typeToJavaType(paramType);
-                paramTypes.add(paramType);
-                parameters.add(new Parameter(location, paramNameStrs.get(param), paramType));
-            } catch (IllegalArgumentException exception) {
+            if (paramType == null) {
                 throw createError(new IllegalArgumentException(
                     "Illegal parameter type [" + this.paramTypeStrs.get(param) + "] for function [" + name + "]."));
             }
+
+            paramClasses[param] = PainlessLookupUtility.typeToJavaType(paramType);
+            paramTypes.add(paramType);
+            parameters.add(new Parameter(location, paramNameStrs.get(param), paramType));
         }
 
-        org.objectweb.asm.commons.Method method = new org.objectweb.asm.commons.Method(name, MethodType.methodType(
-                PainlessLookupUtility.typeToJavaType(rtnType), paramClasses).toMethodDescriptorString());
-        this.method = new PainlessMethod(name, null, null, rtnType, paramTypes, method, Modifier.STATIC | Modifier.PRIVATE, null);
+        typeParameters = paramTypes;
+        methodType = MethodType.methodType(PainlessLookupUtility.typeToJavaType(returnType), paramClasses);
+        method = new org.objectweb.asm.commons.Method(name, MethodType.methodType(
+                PainlessLookupUtility.typeToJavaType(returnType), paramClasses).toMethodDescriptorString());
     }
 
     @Override
@@ -175,7 +172,7 @@ void analyze(Locals locals) {
             allEscape = statement.allEscape;
         }
 
-        if (!methodEscape && rtnType != void.class) {
+        if (!methodEscape && returnType != void.class) {
             throw createError(new IllegalArgumentException("Not all paths provide a return value for method [" + name + "]."));
         }
 
@@ -190,7 +187,7 @@ void write (ClassVisitor writer, CompilerSettings settings, Globals globals) {
         if (synthetic) {
             access |= Opcodes.ACC_SYNTHETIC;
         }
-        final MethodWriter function = new MethodWriter(access, method.method, writer, globals.getStatements(), settings);
+        final MethodWriter function = new MethodWriter(access, method, writer, globals.getStatements(), settings);
         function.visitCode();
         write(function, globals);
         function.endMethod();
@@ -210,25 +207,12 @@ void write(MethodWriter function, Globals globals) {
         }
 
         if (!methodEscape) {
-            if (rtnType == void.class) {
+            if (returnType == void.class) {
                 function.returnValue();
             } else {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
-
-        String staticHandleFieldName = Def.getUserFunctionHandleFieldName(name, parameters.size());
-        globals.addConstantInitializer(new Constant(location, WriterConstants.METHOD_HANDLE_TYPE,
-                                                    staticHandleFieldName, this::initializeConstant));
-    }
-
-    private void initializeConstant(MethodWriter writer) {
-        final Handle handle = new Handle(Opcodes.H_INVOKESTATIC,
-                CLASS_TYPE.getInternalName(),
-                name,
-                method.method.getDescriptor(),
-                false);
-        writer.push(handle);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index c354e78a961a3..0f7445a38c44c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.painless.Constant;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.LocalMethod;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -30,8 +31,6 @@
 import org.elasticsearch.painless.SimpleChecksAdapter;
 import org.elasticsearch.painless.WriterConstants;
 import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.node.SFunction.FunctionReserved;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
@@ -70,6 +69,7 @@
 import static org.elasticsearch.painless.WriterConstants.GET_NAME_METHOD;
 import static org.elasticsearch.painless.WriterConstants.GET_SOURCE_METHOD;
 import static org.elasticsearch.painless.WriterConstants.GET_STATEMENTS_METHOD;
+import static org.elasticsearch.painless.WriterConstants.MAP_TYPE;
 import static org.elasticsearch.painless.WriterConstants.OUT_OF_MEMORY_ERROR_TYPE;
 import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE;
 import static org.elasticsearch.painless.WriterConstants.PAINLESS_EXPLAIN_ERROR_GET_HEADERS_METHOD;
@@ -164,27 +164,31 @@ void extractVariables(Set<String> variables) {
         throw new IllegalStateException("Illegal tree structure.");
     }
 
-    public void analyze(PainlessLookup painlessLookup) {
-        Map<String, PainlessMethod> methods = new HashMap<>();
+    public Map<String, LocalMethod> analyze(PainlessLookup painlessLookup) {
+        Map<String, LocalMethod> methods = new HashMap<>();
 
         for (SFunction function : functions) {
             function.generateSignature(painlessLookup);
 
-            String key = PainlessLookupUtility.buildPainlessMethodKey(function.name, function.parameters.size());
+            String key = Locals.buildLocalMethodKey(function.name, function.parameters.size());
 
-            if (methods.put(key, function.method) != null) {
+            if (methods.put(key,
+                    new LocalMethod(function.name, function.returnType, function.typeParameters, function.methodType)) != null) {
                 throw createError(new IllegalArgumentException("Duplicate functions with name [" + function.name + "]."));
             }
         }
 
-        analyze(Locals.newProgramScope(painlessLookup, methods.values()));
+        Locals locals = Locals.newProgramScope(painlessLookup, methods.values());
+        analyze(locals);
+
+        return locals.getMethods();
     }
 
     @Override
     void analyze(Locals program) {
         for (SFunction function : functions) {
             Locals functionLocals =
-                Locals.newFunctionScope(program, function.rtnType, function.parameters, function.reserved.getMaxLoopCounter());
+                Locals.newFunctionScope(program, function.returnType, function.parameters, function.reserved.getMaxLoopCounter());
             function.analyze(functionLocals);
         }
 
@@ -253,6 +257,7 @@ public void write() {
                 globals.getStatements(), settings);
         bootstrapDef.visitCode();
         bootstrapDef.getStatic(CLASS_TYPE, "$DEFINITION", DEFINITION_TYPE);
+        bootstrapDef.getStatic(CLASS_TYPE, "$LOCALS", MAP_TYPE);
         bootstrapDef.loadArgs();
         bootstrapDef.invokeStatic(DEF_BOOTSTRAP_DELEGATE_TYPE, DEF_BOOTSTRAP_DELEGATE_METHOD);
         bootstrapDef.returnValue();
@@ -263,8 +268,9 @@ public void write() {
         visitor.visitField(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, "$SOURCE", STRING_TYPE.getDescriptor(), null, null).visitEnd();
         visitor.visitField(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, "$STATEMENTS", BITSET_TYPE.getDescriptor(), null, null).visitEnd();
 
-        // Write the static variable used by the method to bootstrap def calls
+        // Write the static variables used by the method to bootstrap def calls
         visitor.visitField(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, "$DEFINITION", DEFINITION_TYPE.getDescriptor(), null, null).visitEnd();
+        visitor.visitField(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, "$LOCALS", MAP_TYPE.getDescriptor(), null, null).visitEnd();
 
         org.objectweb.asm.commons.Method init;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index 12e3154eb562e..46dfa056874f2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -40,6 +40,7 @@
 import static org.elasticsearch.painless.WriterConstants.ITERATOR_HASNEXT;
 import static org.elasticsearch.painless.WriterConstants.ITERATOR_NEXT;
 import static org.elasticsearch.painless.WriterConstants.ITERATOR_TYPE;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
 
 /**
  * Represents a for-each loop for iterables.
@@ -76,12 +77,11 @@ void analyze(Locals locals) {
         if (expression.actual == def.class) {
             method = null;
         } else {
-            method = locals.getPainlessLookup().getPainlessStructFromJavaClass(expression.actual).methods
-                    .get(PainlessLookupUtility.buildPainlessMethodKey("iterator", 0));
+            method = locals.getPainlessLookup().lookupPainlessMethod(expression.actual, false, "iterator", 0);
 
             if (method == null) {
-                throw createError(new IllegalArgumentException("Unable to create iterator for the type " +
-                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(expression.actual) + "]."));
+                    throw createError(new IllegalArgumentException(
+                            "method [" + typeToCanonicalTypeName(expression.actual) + ", iterator/0] not found"));
             }
         }
 
@@ -99,7 +99,7 @@ void write(MethodWriter writer, Globals globals) {
                     .getMethodType(org.objectweb.asm.Type.getType(Iterator.class), org.objectweb.asm.Type.getType(Object.class));
             writer.invokeDefCall("iterator", methodType, DefBootstrap.ITERATOR);
         } else {
-            method.write(writer);
+            writer.invokeMethodCall(method);
         }
 
         writer.visitVarInsn(MethodWriter.getType(iterator.clazz).getOpcode(Opcodes.ISTORE), iterator.getSlot());
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt
index 8491d15c27eb9..f424f6acf25aa 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt
@@ -77,8 +77,8 @@ class org.elasticsearch.index.fielddata.ScriptDocValues$Longs {
 }
 
 class org.elasticsearch.index.fielddata.ScriptDocValues$Dates {
-  org.joda.time.ReadableDateTime get(int)
-  org.joda.time.ReadableDateTime getValue()
+  Object get(int)
+  Object getValue()
   List getValues()
 }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java
index 34bc2c78de662..58864d73c4120 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java
@@ -35,8 +35,8 @@ private static void assertCast(Class<?> actual, Class<?> expected, boolean mustB
         }
 
         PainlessCast cast = AnalyzerCaster.getLegalCast(location, actual, expected, true, false);
-        assertEquals(actual, cast.from);
-        assertEquals(expected, cast.to);
+        assertEquals(actual, cast.originalType);
+        assertEquals(expected, cast.targetType);
 
         if (mustBeExplicit) {
             ClassCastException error = expectThrows(ClassCastException.class,
@@ -44,8 +44,8 @@ private static void assertCast(Class<?> actual, Class<?> expected, boolean mustB
             assertTrue(error.getMessage().startsWith("Cannot cast"));
         } else {
             cast = AnalyzerCaster.getLegalCast(location, actual, expected, false, false);
-            assertEquals(actual, cast.from);
-            assertEquals(expected, cast.to);
+            assertEquals(actual, cast.originalType);
+            assertEquals(expected, cast.targetType);
         }
     }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
index c852d5a41dec1..c68302bde56f2 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
@@ -69,7 +69,7 @@ public Map<String, Object> getTestMap() {
     }
 
     public void testGets() {
-        Compiler compiler = new Compiler(Gets.class, painlessLookup);
+        Compiler compiler = new Compiler(Gets.class, null, null, painlessLookup);
         Map<String, Object> map = new HashMap<>();
         map.put("s", 1);
 
@@ -87,7 +87,7 @@ public abstract static class NoArgs {
         public abstract Object execute();
     }
     public void testNoArgs() {
-        Compiler compiler = new Compiler(NoArgs.class, painlessLookup);
+        Compiler compiler = new Compiler(NoArgs.class, null, null, painlessLookup);
         assertEquals(1, ((NoArgs)scriptEngine.compile(compiler, null, "1", emptyMap())).execute());
         assertEquals("foo", ((NoArgs)scriptEngine.compile(compiler, null, "'foo'", emptyMap())).execute());
 
@@ -111,13 +111,13 @@ public abstract static class OneArg {
         public abstract Object execute(Object arg);
     }
     public void testOneArg() {
-        Compiler compiler = new Compiler(OneArg.class, painlessLookup);
+        Compiler compiler = new Compiler(OneArg.class, null, null, painlessLookup);
         Object rando = randomInt();
         assertEquals(rando, ((OneArg)scriptEngine.compile(compiler, null, "arg", emptyMap())).execute(rando));
         rando = randomAlphaOfLength(5);
         assertEquals(rando, ((OneArg)scriptEngine.compile(compiler, null, "arg", emptyMap())).execute(rando));
 
-        Compiler noargs = new Compiler(NoArgs.class, painlessLookup);
+        Compiler noargs = new Compiler(NoArgs.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, () ->
                 scriptEngine.compile(noargs, null, "doc", emptyMap()));
         assertEquals("Variable [doc] is not defined.", e.getMessage());
@@ -132,7 +132,7 @@ public abstract static class ArrayArg {
         public abstract Object execute(String[] arg);
     }
     public void testArrayArg() {
-        Compiler compiler = new Compiler(ArrayArg.class, painlessLookup);
+        Compiler compiler = new Compiler(ArrayArg.class, null, null, painlessLookup);
         String rando = randomAlphaOfLength(5);
         assertEquals(rando, ((ArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new String[] {rando, "foo"}));
     }
@@ -142,7 +142,7 @@ public abstract static class PrimitiveArrayArg {
         public abstract Object execute(int[] arg);
     }
     public void testPrimitiveArrayArg() {
-        Compiler compiler = new Compiler(PrimitiveArrayArg.class, painlessLookup);
+        Compiler compiler = new Compiler(PrimitiveArrayArg.class, null, null, painlessLookup);
         int rando = randomInt();
         assertEquals(rando, ((PrimitiveArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new int[] {rando, 10}));
     }
@@ -152,7 +152,7 @@ public abstract static class DefArrayArg {
         public abstract Object execute(Object[] arg);
     }
     public void testDefArrayArg() {
-        Compiler compiler = new Compiler(DefArrayArg.class, painlessLookup);
+        Compiler compiler = new Compiler(DefArrayArg.class, null, null, painlessLookup);
         Object rando = randomInt();
         assertEquals(rando, ((DefArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new Object[] {rando, 10}));
         rando = randomAlphaOfLength(5);
@@ -170,7 +170,7 @@ public abstract static class ManyArgs {
         public abstract boolean needsD();
     }
     public void testManyArgs() {
-        Compiler compiler = new Compiler(ManyArgs.class, painlessLookup);
+        Compiler compiler = new Compiler(ManyArgs.class, null, null, painlessLookup);
         int rando = randomInt();
         assertEquals(rando, ((ManyArgs)scriptEngine.compile(compiler, null, "a", emptyMap())).execute(rando, 0, 0, 0));
         assertEquals(10, ((ManyArgs)scriptEngine.compile(compiler, null, "a + b + c + d", emptyMap())).execute(1, 2, 3, 4));
@@ -198,7 +198,7 @@ public abstract static class VarargTest {
         public abstract Object execute(String... arg);
     }
     public void testVararg() {
-        Compiler compiler = new Compiler(VarargTest.class, painlessLookup);
+        Compiler compiler = new Compiler(VarargTest.class, null, null, painlessLookup);
         assertEquals("foo bar baz", ((VarargTest)scriptEngine.compile(compiler, null, "String.join(' ', Arrays.asList(arg))", emptyMap()))
                     .execute("foo", "bar", "baz"));
     }
@@ -214,7 +214,7 @@ public Object executeWithASingleOne(int a, int b, int c) {
         }
     }
     public void testDefaultMethods() {
-        Compiler compiler = new Compiler(DefaultMethods.class, painlessLookup);
+        Compiler compiler = new Compiler(DefaultMethods.class, null, null, painlessLookup);
         int rando = randomInt();
         assertEquals(rando, ((DefaultMethods)scriptEngine.compile(compiler, null, "a", emptyMap())).execute(rando, 0, 0, 0));
         assertEquals(rando, ((DefaultMethods)scriptEngine.compile(compiler, null, "a", emptyMap())).executeWithASingleOne(rando, 0, 0));
@@ -228,7 +228,7 @@ public abstract static class ReturnsVoid {
         public abstract void execute(Map<String, Object> map);
     }
     public void testReturnsVoid() {
-        Compiler compiler = new Compiler(ReturnsVoid.class, painlessLookup);
+        Compiler compiler = new Compiler(ReturnsVoid.class, null, null, painlessLookup);
         Map<String, Object> map = new HashMap<>();
         ((ReturnsVoid)scriptEngine.compile(compiler, null, "map.a = 'foo'", emptyMap())).execute(map);
         assertEquals(singletonMap("a", "foo"), map);
@@ -247,7 +247,7 @@ public abstract static class ReturnsPrimitiveBoolean {
         public abstract boolean execute();
     }
     public void testReturnsPrimitiveBoolean() {
-        Compiler compiler = new Compiler(ReturnsPrimitiveBoolean.class, painlessLookup);
+        Compiler compiler = new Compiler(ReturnsPrimitiveBoolean.class, null, null, painlessLookup);
 
         assertEquals(true, ((ReturnsPrimitiveBoolean)scriptEngine.compile(compiler, null, "true", emptyMap())).execute());
         assertEquals(false, ((ReturnsPrimitiveBoolean)scriptEngine.compile(compiler, null, "false", emptyMap())).execute());
@@ -289,7 +289,7 @@ public abstract static class ReturnsPrimitiveInt {
         public abstract int execute();
     }
     public void testReturnsPrimitiveInt() {
-        Compiler compiler = new Compiler(ReturnsPrimitiveInt.class, painlessLookup);
+        Compiler compiler = new Compiler(ReturnsPrimitiveInt.class, null, null, painlessLookup);
 
         assertEquals(1, ((ReturnsPrimitiveInt)scriptEngine.compile(compiler, null, "1", emptyMap())).execute());
         assertEquals(1, ((ReturnsPrimitiveInt)scriptEngine.compile(compiler, null, "(int) 1L", emptyMap())).execute());
@@ -331,7 +331,7 @@ public abstract static class ReturnsPrimitiveFloat {
         public abstract float execute();
     }
     public void testReturnsPrimitiveFloat() {
-        Compiler compiler = new Compiler(ReturnsPrimitiveFloat.class, painlessLookup);
+        Compiler compiler = new Compiler(ReturnsPrimitiveFloat.class, null, null, painlessLookup);
 
         assertEquals(1.1f, ((ReturnsPrimitiveFloat)scriptEngine.compile(compiler, null, "1.1f", emptyMap())).execute(), 0);
         assertEquals(1.1f, ((ReturnsPrimitiveFloat)scriptEngine.compile(compiler, null, "(float) 1.1d", emptyMap())).execute(), 0);
@@ -362,7 +362,7 @@ public abstract static class ReturnsPrimitiveDouble {
         public abstract double execute();
     }
     public void testReturnsPrimitiveDouble() {
-        Compiler compiler = new Compiler(ReturnsPrimitiveDouble.class, painlessLookup);
+        Compiler compiler = new Compiler(ReturnsPrimitiveDouble.class, null, null, painlessLookup);
 
         assertEquals(1.0, ((ReturnsPrimitiveDouble)scriptEngine.compile(compiler, null, "1", emptyMap())).execute(), 0);
         assertEquals(1.0, ((ReturnsPrimitiveDouble)scriptEngine.compile(compiler, null, "1L", emptyMap())).execute(), 0);
@@ -396,7 +396,7 @@ public abstract static class NoArgumentsConstant {
         public abstract Object execute(String foo);
     }
     public void testNoArgumentsConstant() {
-        Compiler compiler = new Compiler(NoArgumentsConstant.class, painlessLookup);
+        Compiler compiler = new Compiler(NoArgumentsConstant.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertThat(e.getMessage(), startsWith(
@@ -409,7 +409,7 @@ public abstract static class WrongArgumentsConstant {
         public abstract Object execute(String foo);
     }
     public void testWrongArgumentsConstant() {
-        Compiler compiler = new Compiler(WrongArgumentsConstant.class, painlessLookup);
+        Compiler compiler = new Compiler(WrongArgumentsConstant.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertThat(e.getMessage(), startsWith(
@@ -422,7 +422,7 @@ public abstract static class WrongLengthOfArgumentConstant {
         public abstract Object execute(String foo);
     }
     public void testWrongLengthOfArgumentConstant() {
-        Compiler compiler = new Compiler(WrongLengthOfArgumentConstant.class, painlessLookup);
+        Compiler compiler = new Compiler(WrongLengthOfArgumentConstant.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertThat(e.getMessage(), startsWith("[" + WrongLengthOfArgumentConstant.class.getName() + "#ARGUMENTS] has length [2] but ["
@@ -434,7 +434,7 @@ public abstract static class UnknownArgType {
         public abstract Object execute(UnknownArgType foo);
     }
     public void testUnknownArgType() {
-        Compiler compiler = new Compiler(UnknownArgType.class, painlessLookup);
+        Compiler compiler = new Compiler(UnknownArgType.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertEquals("[foo] is of unknown type [" + UnknownArgType.class.getName() + ". Painless interfaces can only accept arguments "
@@ -446,7 +446,7 @@ public abstract static class UnknownReturnType {
         public abstract UnknownReturnType execute(String foo);
     }
     public void testUnknownReturnType() {
-        Compiler compiler = new Compiler(UnknownReturnType.class, painlessLookup);
+        Compiler compiler = new Compiler(UnknownReturnType.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertEquals("Painless can only implement execute methods returning a whitelisted type but [" + UnknownReturnType.class.getName()
@@ -458,7 +458,7 @@ public abstract static class UnknownArgTypeInArray {
         public abstract Object execute(UnknownArgTypeInArray[] foo);
     }
     public void testUnknownArgTypeInArray() {
-        Compiler compiler = new Compiler(UnknownArgTypeInArray.class, painlessLookup);
+        Compiler compiler = new Compiler(UnknownArgTypeInArray.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertEquals("[foo] is of unknown type [" + UnknownArgTypeInArray.class.getName() + ". Painless interfaces can only accept "
@@ -470,7 +470,7 @@ public abstract static class TwoExecuteMethods {
         public abstract Object execute(boolean foo);
     }
     public void testTwoExecuteMethods() {
-        Compiler compiler = new Compiler(TwoExecuteMethods.class, painlessLookup);
+        Compiler compiler = new Compiler(TwoExecuteMethods.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "null", emptyMap()));
         assertEquals("Painless can only implement interfaces that have a single method named [execute] but ["
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java
new file mode 100644
index 0000000000000..15eed75bcb8df
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+/**
+ * These tests run the Painless scripts used in the context docs against
+ * slightly modified data designed around unit tests rather than a fully-
+ * running Elasticsearch server.
+ */
+public class ContextExampleTests extends ScriptTestCase {
+
+    // **** Docs Generator Code ****
+
+    /*
+
+    import java.io.FileWriter;
+    import java.io.IOException;
+
+    public class Generator {
+
+    public final static String[] theatres = new String[] {"Down Port", "Graye", "Skyline", "Courtyard"};
+    public final static String[] plays = new String[] {"Driving", "Pick It Up", "Sway and Pull", "Harriot",
+            "The Busline", "Ants Underground", "Exploria", "Line and Single", "Shafted", "Sunnyside Down",
+            "Test Run", "Auntie Jo"};
+    public final static String[] actors = new String[] {"James Holland", "Krissy Smith", "Joe Muir", "Ryan Earns",
+            "Joel Madigan", "Jessica Brown", "Baz Knight", "Jo Hangum", "Rachel Grass", "Phoebe Miller", "Sarah Notch",
+            "Brayden Green", "Joshua Iller", "Jon Hittle", "Rob Kettleman", "Laura Conrad", "Simon Hower", "Nora Blue",
+            "Mike Candlestick", "Jacey Bell"};
+
+    public static void writeSeat(FileWriter writer, int id, String theatre, String play, String[] actors,
+                                 String date, String time, int row, int number, double cost, boolean sold) throws IOException {
+        StringBuilder builder = new StringBuilder();
+        builder.append("{ \"create\" : { \"_index\" : \"seats\", \"_type\" : \"seat\", \"_id\" : \"");
+        builder.append(id);
+        builder.append("\" } }\n");
+        builder.append("{ \"theatre\" : \"");
+        builder.append(theatre);
+        builder.append("\", \"play\" : \"");
+        builder.append(play);
+        builder.append("\", \"actors\": [ \"");
+        for (String actor : actors) {
+            builder.append(actor);
+            if (actor.equals(actors[actors.length - 1]) == false) {
+                builder.append("\", \"");
+            }
+        }
+        builder.append("\" ], \"date\": \"");
+        builder.append(date);
+        builder.append("\", \"time\": \"");
+        builder.append(time);
+        builder.append("\", \"row\": ");
+        builder.append(row);
+        builder.append(", \"number\": ");
+        builder.append(number);
+        builder.append(", \"cost\": ");
+        builder.append(cost);
+        builder.append(", \"sold\": ");
+        builder.append(sold ? "true" : "false");
+        builder.append(" }\n");
+        writer.write(builder.toString());
+    }
+
+    public static void main(String args[]) throws IOException {
+        FileWriter writer = new FileWriter("/home/jdconrad/test/seats.json");
+        int id = 0;
+
+        for (int playCount = 0; playCount < 12; ++playCount) {
+            String play = plays[playCount];
+            String theatre;
+            String[] actor;
+            int startMonth;
+            int endMonth;
+            String time;
+
+            if (playCount == 0) {
+                theatre = theatres[0];
+                actor = new String[] {actors[0], actors[1], actors[2], actors[3]};
+                startMonth = 4;
+                endMonth = 5;
+                time = "3:00PM";
+            } else if (playCount == 1) {
+                theatre = theatres[0];
+                actor = new String[] {actors[4], actors[5], actors[6], actors[7], actors[8], actors[9]};
+                startMonth = 4;
+                endMonth = 6;
+                time = "8:00PM";
+            } else if (playCount == 2) {
+                theatre = theatres[0];
+                actor = new String[] {actors[0], actors[1], actors[2], actors[3],
+                                      actors[4], actors[5], actors[6], actors[7]};
+                startMonth = 6;
+                endMonth = 8;
+                time = "3:00 PM";
+            } else if (playCount == 3) {
+                theatre = theatres[0];
+                actor = new String[] {actors[9], actors[10], actors[11], actors[12], actors[13], actors[14],
+                                      actors[15], actors[16], actors[17], actors[18], actors[19]};
+                startMonth = 7;
+                endMonth = 8;
+                time = "8:00PM";
+            } else if (playCount == 4) {
+                theatre = theatres[0];
+                actor = new String[] {actors[13], actors[14], actors[15], actors[17], actors[18], actors[19]};
+                startMonth = 8;
+                endMonth = 10;
+                time = "3:00PM";
+            } else if (playCount == 5) {
+                theatre = theatres[0];
+                actor = new String[] {actors[8], actors[9], actors[10], actors[11], actors[12]};
+                startMonth = 8;
+                endMonth = 10;
+                time = "8:00PM";
+            } else if (playCount == 6) {
+                theatre = theatres[1];
+                actor = new String[] {actors[10], actors[11], actors[12], actors[13], actors[14], actors[15], actors[16]};
+                startMonth = 4;
+                endMonth = 5;
+                time = "11:00AM";
+            } else if (playCount == 7) {
+                theatre = theatres[1];
+                actor = new String[] {actors[17], actors[18]};
+                startMonth = 6;
+                endMonth = 9;
+                time = "2:00PM";
+            } else if (playCount == 8) {
+                theatre = theatres[1];
+                actor = new String[] {actors[0], actors[1], actors[2], actors[3], actors[16]};
+                startMonth = 10;
+                endMonth = 11;
+                time = "11:00AM";
+            } else if (playCount == 9) {
+                theatre = theatres[2];
+                actor = new String[] {actors[1], actors[2], actors[3], actors[17], actors[18], actors[19]};
+                startMonth = 3;
+                endMonth = 6;
+                time = "4:00PM";
+            } else if (playCount == 10) {
+                theatre = theatres[2];
+                actor = new String[] {actors[2], actors[3], actors[4], actors[5]};
+                startMonth = 7;
+                endMonth = 8;
+                time = "7:30PM";
+            } else if (playCount == 11) {
+                theatre = theatres[2];
+                actor = new String[] {actors[7], actors[13], actors[14], actors[15], actors[16], actors[17]};
+                startMonth = 9;
+                endMonth = 12;
+                time = "5:40PM";
+            } else {
+                throw new RuntimeException("too many plays");
+            }
+
+            int rows;
+            int number;
+
+            if (playCount < 6) {
+               rows = 3;
+               number = 12;
+            } else if (playCount < 9) {
+               rows = 5;
+               number = 9;
+            } else if (playCount < 12) {
+               rows = 11;
+               number = 15;
+            } else {
+                throw new RuntimeException("too many seats");
+            }
+
+            for (int month = startMonth; month <= endMonth; ++month) {
+                for (int day = 1; day <= 14; ++day) {
+                    for (int row = 1; row <= rows; ++row) {
+                        for (int count = 1; count <= number; ++count) {
+                            String date = "2018-" + month + "-" + day;
+                            double cost = (25 - row) * 1.25;
+
+                            writeSeat(writer, ++id, theatre, play, actor, date, time, row, count, cost, false);
+                        }
+                    }
+                }
+            }
+        }
+
+        writer.write("\n");
+        writer.close();
+        }
+    }
+
+    */
+
+    // **** Initial Mappings ****
+
+    /*
+
+    curl -X PUT "localhost:9200/seats" -H 'Content-Type: application/json' -d'
+    {
+      "mappings": {
+        "seat": {
+          "properties": {
+            "theatre":  { "type": "keyword" },
+            "play":     { "type": "text"    },
+            "actors":   { "type": "text"    },
+            "row":      { "type": "integer" },
+            "number":   { "type": "integer" },
+            "cost":     { "type": "double"  },
+            "sold":     { "type": "boolean" },
+            "datetime": { "type": "date"    },
+            "date":     { "type": "keyword" },
+            "time":     { "type": "keyword" }
+          }
+        }
+      }
+    }
+    '
+
+    */
+
+    // Create Ingest to Modify Dates:
+
+    /*
+
+    curl -X PUT "localhost:9200/_ingest/pipeline/seats" -H 'Content-Type: application/json' -d'
+    {
+        "description": "update datetime for seats",
+        "processors": [
+          {
+            "script": {
+              "source": "String[] split(String s, char d) { int count = 0; for (char c : s.toCharArray()) { if (c == d) { ++count; } } if (count == 0) { return new String[] {s}; } String[] r = new String[count + 1]; int i0 = 0, i1 = 0; count = 0; for (char c : s.toCharArray()) { if (c == d) { r[count++] = s.substring(i0, i1); i0 = i1 + 1; } ++i1; } r[count] = s.substring(i0, i1); return r; } String[] dateSplit = split(ctx.date, (char)\"-\"); String year = dateSplit[0].trim(); String month = dateSplit[1].trim(); if (month.length() == 1) { month = \"0\" + month; } String day = dateSplit[2].trim(); if (day.length() == 1) { day = \"0\" + day; } boolean pm = ctx.time.substring(ctx.time.length() - 2).equals(\"PM\"); String[] timeSplit = split(ctx.time.substring(0, ctx.time.length() - 2), (char)\":\"); int hours = Integer.parseInt(timeSplit[0].trim()); int minutes = Integer.parseInt(timeSplit[1].trim()); if (pm) { hours += 12; } String dts = year + \"-\" + month + \"-\" + day + \"T\" + (hours < 10 ? \"0\" + hours : \"\" + hours) + \":\" + (minutes < 10 ? \"0\" + minutes : \"\" + minutes) + \":00+08:00\"; ZonedDateTime dt = ZonedDateTime.parse(dts, DateTimeFormatter.ISO_OFFSET_DATE_TIME); ctx.datetime = dt.getLong(ChronoField.INSTANT_SECONDS)*1000L;"
+            }
+          }
+        ]
+    }
+    '
+
+    */
+
+    public void testIngestProcessorScript() {
+        assertEquals(1535785200000L,
+            exec("String[] split(String s, char d) {" +
+                "    int count = 0;" +
+                "    for (char c : s.toCharArray()) {" +
+                "        if (c == d) {" +
+                "            ++count;" +
+                "        }" +
+                "    }" +
+                "    if (count == 0) {" +
+                "        return new String[] {s};" +
+                "    }" +
+                "    String[] r = new String[count + 1];" +
+                "    int i0 = 0, i1 = 0;" +
+                "    count = 0;" +
+                "    for (char c : s.toCharArray()) {" +
+                "        if (c == d) {" +
+                "            r[count++] = s.substring(i0, i1);" +
+                "            i0 = i1 + 1;" +
+                "        }" +
+                "        ++i1;" +
+                "    }" +
+                "    r[count] = s.substring(i0, i1);" +
+                "    return r;" +
+                "}" +
+                "def x = ['date': '2018-9-1', 'time': '3:00 PM'];" +
+                "String[] dateSplit = split(x.date, (char)'-');" +
+                "String year = dateSplit[0].trim();" +
+                "String month = dateSplit[1].trim();" +
+                "if (month.length() == 1) {" +
+                "    month = '0' + month;" +
+                "}" +
+                "String day = dateSplit[2].trim();" +
+                "if (day.length() == 1) {" +
+                "    day = '0' + day;" +
+                "}" +
+                "boolean pm = x.time.substring(x.time.length() - 2).equals('PM');" +
+                "String[] timeSplit = split(x.time.substring(0, x.time.length() - 2), (char)':');" +
+                "int hours = Integer.parseInt(timeSplit[0].trim());" +
+                "String minutes = timeSplit[1].trim();" +
+                "if (pm) {" +
+                "    hours += 12;" +
+                "}" +
+                "String dts = year + '-' + month + '-' + day + " +
+                "'T' + (hours < 10 ? '0' + hours : '' + hours) + ':' + minutes + ':00+08:00';" +
+                "ZonedDateTime dt = ZonedDateTime.parse(dts, DateTimeFormatter.ISO_OFFSET_DATE_TIME);" +
+                "return dt.getLong(ChronoField.INSTANT_SECONDS) * 1000L"
+            )
+        );
+    }
+
+    // Post Generated Data:
+
+    /*
+
+    curl -XPOST localhost:9200/seats/seat/_bulk?pipeline=seats -H "Content-Type: application/x-ndjson" --data-binary "@/home/jdconrad/test/seats.json"
+
+    */
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
index 48af3898e0952..ae33ebfb6e9c5 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
@@ -40,7 +40,7 @@ static String toString(Class<?> iface, String source, CompilerSettings settings)
         PrintWriter outputWriter = new PrintWriter(output);
         Textifier textifier = new Textifier();
         try {
-            new Compiler(iface, PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS))
+            new Compiler(iface, null, null, PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS))
                     .compile("<debugging>", source, settings, textifier);
         } catch (RuntimeException e) {
             textifier.print(outputWriter);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index 1ef855d561c52..88d257a067205 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -38,6 +38,7 @@ public class DefBootstrapTests extends ESTestCase {
     /** calls toString() on integers, twice */
     public void testOneType() throws Throwable {
         CallSite site = DefBootstrap.bootstrap(painlessLookup,
+                                               Collections.emptyMap(),
                                                MethodHandles.publicLookup(),
                                                "toString",
                                                MethodType.methodType(String.class, Object.class),
@@ -58,6 +59,7 @@ public void testOneType() throws Throwable {
 
     public void testTwoTypes() throws Throwable {
         CallSite site = DefBootstrap.bootstrap(painlessLookup,
+                                               Collections.emptyMap(),
                                                MethodHandles.publicLookup(),
                                                "toString",
                                                MethodType.methodType(String.class, Object.class),
@@ -83,6 +85,7 @@ public void testTooManyTypes() throws Throwable {
         // if this changes, test must be rewritten
         assertEquals(5, DefBootstrap.PIC.MAX_DEPTH);
         CallSite site = DefBootstrap.bootstrap(painlessLookup,
+                                               Collections.emptyMap(),
                                                MethodHandles.publicLookup(),
                                                "toString",
                                                MethodType.methodType(String.class, Object.class),
@@ -109,6 +112,7 @@ public void testTooManyTypes() throws Throwable {
     /** test that we revert to the megamorphic classvalue cache and that it works as expected */
     public void testMegamorphic() throws Throwable {
         DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(painlessLookup,
+                                                                          Collections.emptyMap(),
                                                                           MethodHandles.publicLookup(),
                                                                           "size",
                                                                           MethodType.methodType(int.class, Object.class),
@@ -130,7 +134,7 @@ public void testMegamorphic() throws Throwable {
         final IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> {
             Integer.toString((int)handle.invokeExact(new Object()));
         });
-        assertEquals("Unable to find dynamic method [size] with [0] arguments for class [java.lang.Object].", iae.getMessage());
+        assertEquals("dynamic method [java.lang.Object, size/0] not found", iae.getMessage());
         assertTrue("Does not fail inside ClassValue.computeValue()", Arrays.stream(iae.getStackTrace()).anyMatch(e -> {
             return e.getMethodName().equals("computeValue") &&
                    e.getClassName().startsWith("org.elasticsearch.painless.DefBootstrap$PIC$");
@@ -141,6 +145,7 @@ public void testMegamorphic() throws Throwable {
 
     public void testNullGuardAdd() throws Throwable {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup,
+                                                                          Collections.emptyMap(),
                                                                           MethodHandles.publicLookup(),
                                                                           "add",
                                                                           MethodType.methodType(Object.class, Object.class, Object.class),
@@ -153,6 +158,7 @@ public void testNullGuardAdd() throws Throwable {
 
     public void testNullGuardAddWhenCached() throws Throwable {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup,
+                                                                          Collections.emptyMap(),
                                                                           MethodHandles.publicLookup(),
                                                                           "add",
                                                                           MethodType.methodType(Object.class, Object.class, Object.class),
@@ -166,6 +172,7 @@ public void testNullGuardAddWhenCached() throws Throwable {
 
     public void testNullGuardEq() throws Throwable {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup,
+                                                                          Collections.emptyMap(),
                                                                           MethodHandles.publicLookup(),
                                                                           "eq",
                                                                           MethodType.methodType(boolean.class, Object.class, Object.class),
@@ -179,6 +186,7 @@ public void testNullGuardEq() throws Throwable {
 
     public void testNullGuardEqWhenCached() throws Throwable {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup,
+                                                                          Collections.emptyMap(),
                                                                           MethodHandles.publicLookup(),
                                                                           "eq",
                                                                           MethodType.methodType(boolean.class, Object.class, Object.class),
@@ -197,6 +205,7 @@ public void testNullGuardEqWhenCached() throws Throwable {
 
     public void testNoNullGuardAdd() throws Throwable {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup,
+                                                                          Collections.emptyMap(),
                                                                           MethodHandles.publicLookup(),
                                                                           "add",
                                                                           MethodType.methodType(Object.class, int.class, Object.class),
@@ -211,6 +220,7 @@ public void testNoNullGuardAdd() throws Throwable {
 
     public void testNoNullGuardAddWhenCached() throws Throwable {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup,
+                                                                          Collections.emptyMap(),
                                                                           MethodHandles.publicLookup(),
                                                                           "add",
                                                                           MethodType.methodType(Object.class, int.class, Object.class),
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index fd47db6b83d41..5829593f52441 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -27,7 +27,6 @@
 import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
-import static org.hamcrest.Matchers.startsWith;
 
 public class FunctionRefTests extends ScriptTestCase {
 
@@ -193,14 +192,15 @@ public void testMethodMissing() {
         Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("List l = [2, 1]; l.sort(Integer::bogus); return l.get(0);");
         });
-        assertThat(e.getMessage(), startsWith("Unknown reference"));
+        assertThat(e.getMessage(), containsString("function reference [Integer::bogus/2] matching [java.util.Comparator"));
     }
 
     public void testQualifiedMethodMissing() {
         Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("List l = [2, 1]; l.sort(org.joda.time.ReadableDateTime::bogus); return l.get(0);", false);
         });
-        assertThat(e.getMessage(), startsWith("Unknown reference"));
+        assertThat(e.getMessage(),
+                containsString("function reference [org.joda.time.ReadableDateTime::bogus/2] matching [java.util.Comparator"));
     }
 
     public void testClassMissing() {
@@ -223,11 +223,12 @@ public void testNotFunctionalInterface() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("List l = new ArrayList(); l.add(2); l.add(1); l.add(Integer::bogus); return l.get(0);");
         });
-        assertThat(expected.getMessage(), containsString("Cannot convert function reference"));
+        assertThat(expected.getMessage(),
+                containsString("cannot convert function reference [Integer::bogus] to a non-functional interface [def]"));
     }
 
     public void testIncompatible() {
-        expectScriptThrows(BootstrapMethodError.class, () -> {
+        expectScriptThrows(ClassCastException.class, () -> {
             exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(String::startsWith); return l.get(0);");
         });
     }
@@ -236,28 +237,32 @@ public void testWrongArity() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("Optional.empty().orElseGet(String::startsWith);");
         });
-        assertThat(expected.getMessage(), containsString("Unknown reference"));
+        assertThat(expected.getMessage(),
+                containsString("function reference [String::startsWith/0] matching [java.util.function.Supplier"));
     }
 
     public void testWrongArityNotEnough() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(String::isEmpty);");
         });
-        assertTrue(expected.getMessage().contains("Unknown reference"));
+        assertThat(expected.getMessage(), containsString(
+                "function reference [String::isEmpty/2] matching [java.util.Comparator"));
     }
 
     public void testWrongArityDef() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def y = Optional.empty(); return y.orElseGet(String::startsWith);");
         });
-        assertThat(expected.getMessage(), containsString("Unknown reference"));
+        assertThat(expected.getMessage(),
+                containsString("function reference [String::startsWith/0] matching [java.util.function.Supplier"));
     }
 
     public void testWrongArityNotEnoughDef() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def l = new ArrayList(); l.add(2); l.add(1); l.sort(String::isEmpty);");
         });
-        assertThat(expected.getMessage(), containsString("Unknown reference"));
+        assertThat(expected.getMessage(),
+                containsString("function reference [String::isEmpty/2] matching [java.util.Comparator"));
     }
 
     public void testReturnVoid() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index 20e257e574709..1f1a6f95b3608 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -184,7 +184,7 @@ public void testWrongArityDef() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def y = Optional.empty(); return y.orElseGet(x -> x);");
         });
-        assertTrue(expected.getMessage(), expected.getMessage().contains("Incorrect number of parameters"));
+        assertTrue(expected.getMessage(), expected.getMessage().contains("due to an incorrect number of arguments"));
     }
 
     public void testWrongArityNotEnough() {
@@ -200,7 +200,7 @@ public void testWrongArityNotEnoughDef() {
             exec("def l = new ArrayList(); l.add(1); l.add(1); "
                + "return l.stream().mapToInt(() -> 5).sum();");
         });
-        assertTrue(expected.getMessage().contains("Incorrect number of parameters"));
+        assertTrue(expected.getMessage(), expected.getMessage().contains("due to an incorrect number of arguments"));
     }
 
     public void testLambdaInFunction() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
index fce827e686caa..52c28799fae34 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
@@ -23,12 +23,12 @@
 public class OverloadTests extends ScriptTestCase {
 
     public void testMethod() {
-        assertEquals(2, exec("return 'abc123abc'.indexOf('c');"));
-        assertEquals(8, exec("return 'abc123abc'.indexOf('c', 3);"));
+        //assertEquals(2, exec("return 'abc123abc'.indexOf('c');"));
+        //assertEquals(8, exec("return 'abc123abc'.indexOf('c', 3);"));
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("return 'abc123abc'.indexOf('c', 3, 'bogus');");
         });
-        assertTrue(expected.getMessage().contains("[indexOf] with [3] arguments"));
+        assertTrue(expected.getMessage().contains("[java.lang.String, indexOf/3]"));
     }
     
     public void testMethodDynamic() {
@@ -37,7 +37,7 @@ public void testMethodDynamic() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def x = 'abc123abc'; return x.indexOf('c', 3, 'bogus');");
         });
-        assertTrue(expected.getMessage().contains("dynamic method [indexOf]"));
+        assertTrue(expected.getMessage().contains("dynamic method [java.lang.String, indexOf/3] not found"));
     }
     
     public void testConstructor() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
index ff0d423117564..1460d5f2359b6 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessField;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
@@ -44,9 +45,9 @@
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 import static java.util.Comparator.comparing;
-import static java.util.stream.Collectors.toList;
 
 /**
  * Generates an API reference from the method and type whitelists in {@link PainlessLookup}.
@@ -55,9 +56,10 @@ public class PainlessDocGenerator {
 
     private static final PainlessLookup PAINLESS_LOOKUP = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS);
     private static final Logger logger = ESLoggerFactory.getLogger(PainlessDocGenerator.class);
-    private static final Comparator<PainlessField> FIELD_NAME = comparing(f -> f.name);
-    private static final Comparator<PainlessMethod> METHOD_NAME = comparing(m -> m.name);
-    private static final Comparator<PainlessMethod> NUMBER_OF_ARGS = comparing(m -> m.arguments.size());
+    private static final Comparator<PainlessField> FIELD_NAME = comparing(f -> f.javaField.getName());
+    private static final Comparator<PainlessMethod> METHOD_NAME = comparing(m -> m.javaMethod.getName());
+    private static final Comparator<PainlessMethod> METHOD_NUMBER_OF_PARAMS = comparing(m -> m.typeParameters.size());
+    private static final Comparator<PainlessConstructor> CONSTRUCTOR_NUMBER_OF_PARAMS = comparing(m -> m.typeParameters.size());
 
     public static void main(String[] args) throws IOException {
         Path apiRootPath = PathUtils.get(args[0]);
@@ -72,9 +74,10 @@ public static void main(String[] args) throws IOException {
             Files.newOutputStream(indexPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
             false, StandardCharsets.UTF_8.name())) {
             emitGeneratedWarning(indexStream);
-            List<Class<?>> classes = PAINLESS_LOOKUP.getStructs().stream().sorted(comparing(Class::getCanonicalName)).collect(toList());
+            List<Class<?>> classes = PAINLESS_LOOKUP.getClasses().stream().sorted(
+                    Comparator.comparing(Class::getCanonicalName)).collect(Collectors.toList());
             for (Class<?> clazz : classes) {
-                PainlessClass struct = PAINLESS_LOOKUP.getPainlessStructFromJavaClass(clazz);
+                PainlessClass struct = PAINLESS_LOOKUP.lookupPainlessClass(clazz);
                 String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(clazz);
 
                 if (clazz.isPrimitive()) {
@@ -103,16 +106,19 @@ public static void main(String[] args) throws IOException {
 
                     Consumer<PainlessField> documentField = field -> PainlessDocGenerator.documentField(typeStream, field);
                     Consumer<PainlessMethod> documentMethod = method -> PainlessDocGenerator.documentMethod(typeStream, method);
+                    Consumer<PainlessConstructor> documentConstructor =
+                            constructor -> PainlessDocGenerator.documentConstructor(typeStream, constructor);
                     struct.staticFields.values().stream().sorted(FIELD_NAME).forEach(documentField);
                     struct.fields.values().stream().sorted(FIELD_NAME).forEach(documentField);
-                    struct.staticMethods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(documentMethod);
-                    struct.constructors.values().stream().sorted(NUMBER_OF_ARGS).forEach(documentMethod);
+                    struct.staticMethods.values().stream().sorted(
+                            METHOD_NAME.thenComparing(METHOD_NUMBER_OF_PARAMS)).forEach(documentMethod);
+                    struct.constructors.values().stream().sorted(CONSTRUCTOR_NUMBER_OF_PARAMS).forEach(documentConstructor);
                     Map<String, Class<?>> inherited = new TreeMap<>();
-                    struct.methods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(method -> {
-                        if (method.target == clazz) {
+                    struct.methods.values().stream().sorted(METHOD_NAME.thenComparing(METHOD_NUMBER_OF_PARAMS)).forEach(method -> {
+                        if (method.targetClass == clazz) {
                             documentMethod(typeStream, method);
                         } else {
-                            inherited.put(canonicalClassName, method.target);
+                            inherited.put(canonicalClassName, method.targetClass);
                         }
                     });
 
@@ -142,17 +148,17 @@ private static void documentField(PrintStream stream, PainlessField field) {
         emitAnchor(stream, field);
         stream.print("]]");
 
-        if (Modifier.isStatic(field.modifiers)) {
+        if (Modifier.isStatic(field.javaField.getModifiers())) {
             stream.print("static ");
         }
 
-        emitType(stream, field.clazz);
+        emitType(stream, field.typeParameter);
         stream.print(' ');
 
         String javadocRoot = javadocRoot(field);
         emitJavadocLink(stream, javadocRoot, field);
         stream.print('[');
-        stream.print(field.name);
+        stream.print(field.javaField.getName());
         stream.print(']');
 
         if (javadocRoot.equals("java8")) {
@@ -164,6 +170,41 @@ private static void documentField(PrintStream stream, PainlessField field) {
         stream.println();
     }
 
+    /**
+     * Document a constructor.
+     */
+    private static void documentConstructor(PrintStream stream, PainlessConstructor constructor) {
+        stream.print("* ++[[");
+        emitAnchor(stream, constructor);
+        stream.print("]]");
+
+        String javadocRoot = javadocRoot(constructor.javaConstructor.getDeclaringClass());
+        emitJavadocLink(stream, javadocRoot, constructor);
+        stream.print('[');
+
+        stream.print(constructorName(constructor));
+
+        stream.print("](");
+        boolean first = true;
+        for (Class<?> arg : constructor.typeParameters) {
+            if (first) {
+                first = false;
+            } else {
+                stream.print(", ");
+            }
+            emitType(stream, arg);
+        }
+        stream.print(")++");
+
+        if (javadocRoot.equals("java8")) {
+            stream.print(" (");
+            emitJavadocLink(stream, "java9", constructor);
+            stream.print("[java 9])");
+        }
+
+        stream.println();
+    }
+
     /**
      * Document a method.
      */
@@ -172,14 +213,12 @@ private static void documentMethod(PrintStream stream, PainlessMethod method) {
         emitAnchor(stream, method);
         stream.print("]]");
 
-        if (null == method.augmentation && Modifier.isStatic(method.modifiers)) {
+        if (method.targetClass == method.javaMethod.getDeclaringClass() && Modifier.isStatic(method.javaMethod.getModifiers())) {
             stream.print("static ");
         }
 
-        if (false == method.name.equals("<init>")) {
-            emitType(stream, method.rtn);
-            stream.print(' ');
-        }
+        emitType(stream, method.returnType);
+        stream.print(' ');
 
         String javadocRoot = javadocRoot(method);
         emitJavadocLink(stream, javadocRoot, method);
@@ -189,7 +228,7 @@ private static void documentMethod(PrintStream stream, PainlessMethod method) {
 
         stream.print("](");
         boolean first = true;
-        for (Class<?> arg : method.arguments) {
+        for (Class<?> arg : method.typeParameters) {
             if (first) {
                 first = false;
             } else {
@@ -216,28 +255,43 @@ private static void emitAnchor(PrintStream stream, Class<?> clazz) {
         stream.print(PainlessLookupUtility.typeToCanonicalTypeName(clazz).replace('.', '-'));
     }
 
+    /**
+     * Anchor text for a {@link PainlessConstructor}.
+     */
+    private static void emitAnchor(PrintStream stream, PainlessConstructor constructor) {
+        emitAnchor(stream, constructor.javaConstructor.getDeclaringClass());
+        stream.print('-');
+        stream.print(constructorName(constructor));
+        stream.print('-');
+        stream.print(constructor.typeParameters.size());
+    }
+
     /**
      * Anchor text for a {@link PainlessMethod}.
      */
     private static void emitAnchor(PrintStream stream, PainlessMethod method) {
-        emitAnchor(stream, method.target);
+        emitAnchor(stream, method.targetClass);
         stream.print('-');
         stream.print(methodName(method));
         stream.print('-');
-        stream.print(method.arguments.size());
+        stream.print(method.typeParameters.size());
     }
 
     /**
      * Anchor text for a {@link PainlessField}.
      */
     private static void emitAnchor(PrintStream stream, PainlessField field) {
-        emitAnchor(stream, field.target);
+        emitAnchor(stream, field.javaField.getDeclaringClass());
         stream.print('-');
-        stream.print(field.name);
+        stream.print(field.javaField.getName());
+    }
+
+    private static String constructorName(PainlessConstructor constructor) {
+        return PainlessLookupUtility.typeToCanonicalTypeName(constructor.javaConstructor.getDeclaringClass());
     }
 
     private static String methodName(PainlessMethod method) {
-        return method.name.equals("<init>") ? PainlessLookupUtility.typeToCanonicalTypeName(method.target) : method.name;
+        return PainlessLookupUtility.typeToCanonicalTypeName(method.targetClass);
     }
 
     /**
@@ -269,6 +323,34 @@ private static void emitStruct(PrintStream stream, Class<?> clazz) {
         }
     }
 
+    /**
+     * Emit an external link to Javadoc for a {@link PainlessMethod}.
+     *
+     * @param root name of the root uri variable
+     */
+    private static void emitJavadocLink(PrintStream stream, String root, PainlessConstructor constructor) {
+        stream.print("link:{");
+        stream.print(root);
+        stream.print("-javadoc}/");
+        stream.print(classUrlPath(constructor.javaConstructor.getDeclaringClass()));
+        stream.print(".html#");
+        stream.print(constructorName(constructor));
+        stream.print("%2D");
+        boolean first = true;
+        for (Class<?> clazz: constructor.typeParameters) {
+            if (first) {
+                first = false;
+            } else {
+                stream.print("%2D");
+            }
+            stream.print(clazz.getName());
+            if (clazz.isArray()) {
+                stream.print(":A");
+            }
+        }
+        stream.print("%2D");
+    }
+
     /**
      * Emit an external link to Javadoc for a {@link PainlessMethod}.
      *
@@ -278,16 +360,16 @@ private static void emitJavadocLink(PrintStream stream, String root, PainlessMet
         stream.print("link:{");
         stream.print(root);
         stream.print("-javadoc}/");
-        stream.print(classUrlPath(method.augmentation != null ? method.augmentation : method.target));
+        stream.print(classUrlPath(method.javaMethod.getDeclaringClass()));
         stream.print(".html#");
         stream.print(methodName(method));
         stream.print("%2D");
         boolean first = true;
-        if (method.augmentation != null) {
+        if (method.targetClass != method.javaMethod.getDeclaringClass()) {
             first = false;
-            stream.print(method.target.getName());
+            stream.print(method.javaMethod.getDeclaringClass().getName());
         }
-        for (Class<?> clazz: method.arguments) {
+        for (Class<?> clazz: method.typeParameters) {
             if (first) {
                 first = false;
             } else {
@@ -310,26 +392,26 @@ private static void emitJavadocLink(PrintStream stream, String root, PainlessFie
         stream.print("link:{");
         stream.print(root);
         stream.print("-javadoc}/");
-        stream.print(classUrlPath(field.target));
+        stream.print(classUrlPath(field.javaField.getDeclaringClass()));
         stream.print(".html#");
-        stream.print(field.javaName);
+        stream.print(field.javaField.getName());
     }
 
     /**
      * Pick the javadoc root for a {@link PainlessMethod}.
      */
     private static String javadocRoot(PainlessMethod method) {
-        if (method.augmentation != null) {
+        if (method.targetClass != method.javaMethod.getDeclaringClass()) {
             return "painless";
         }
-        return javadocRoot(method.target);
+        return javadocRoot(method.targetClass);
     }
 
     /**
      * Pick the javadoc root for a {@link PainlessField}.
      */
     private static String javadocRoot(PainlessField field) {
-        return javadocRoot(field.target);
+        return javadocRoot(field.javaField.getDeclaringClass());
     }
 
     /**
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
index 911a50468cc17..8143c39ce6f6b 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
@@ -252,7 +252,7 @@ public void testCantUsePatternCompile() {
         IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("Pattern.compile('aa')");
         });
-        assertEquals("Unknown call [compile] with [1] arguments on type [java.util.regex.Pattern].", e.getMessage());
+        assertTrue(e.getMessage().contains("[java.util.regex.Pattern, compile/1]"));
     }
 
     public void testBadRegexPattern() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 8eeb25c9676c7..f2d93aa759d07 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -219,7 +219,7 @@ public void testIllegalDynamicMethod() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def x = 'test'; return x.getClass().toString()");
         });
-        assertTrue(expected.getMessage().contains("Unable to find dynamic method"));
+        assertTrue(expected.getMessage().contains("dynamic method [java.lang.String, getClass/0] not found"));
     }
 
     public void testDynamicNPE() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
index c64014d81a5de..12d57fab11d98 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
@@ -162,12 +162,12 @@ public void testECapturingFunctionRef() {
     public void testECast() {
         Location l = new Location(getTestName(), 0);
         AExpression child = new EConstant(l, "test");
-        PainlessCast cast = PainlessCast.standard(String.class, Integer.class, true);
+        PainlessCast cast = PainlessCast.originalTypetoTargetType(String.class, Integer.class, true);
         assertEquals("(ECast java.lang.Integer (EConstant String 'test'))", new ECast(l, child, cast).toString());
 
         l = new Location(getTestName(), 1);
         child = new EBinary(l, Operation.ADD, new EConstant(l, "test"), new EConstant(l, 12));
-        cast = PainlessCast.standard(Integer.class, Boolean.class, true);
+        cast = PainlessCast.originalTypetoTargetType(Integer.class, Boolean.class, true);
         assertEquals("(ECast java.lang.Boolean (EBinary (EConstant String 'test') + (EConstant Integer 12)))",
             new ECast(l, child, cast).toString());
     }
@@ -404,7 +404,7 @@ public void testPSubBrace() {
 
     public void testPSubCallInvoke() {
         Location l = new Location(getTestName(), 0);
-        PainlessClass c = painlessLookup.getPainlessStructFromJavaClass(Integer.class);
+        PainlessClass c = painlessLookup.lookupPainlessClass(Integer.class);
         PainlessMethod m = c.methods.get(PainlessLookupUtility.buildPainlessMethodKey("toString", 0));
         PSubCallInvoke node = new PSubCallInvoke(l, m, null, emptyList());
         node.prefix = new EVariable(l, "a");
@@ -459,7 +459,7 @@ public void testPSubDefField() {
 
     public void testPSubField() {
         Location l = new Location(getTestName(), 0);
-        PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(Boolean.class);
+        PainlessClass s = painlessLookup.lookupPainlessClass(Boolean.class);
         PainlessField f = s.staticFields.get("TRUE");
         PSubField node = new PSubField(l, f);
         node.prefix = new EStatic(l, "Boolean");
@@ -497,7 +497,7 @@ public void testPSubMapShortcut() {
 
     public void testPSubShortcut() {
         Location l = new Location(getTestName(), 0);
-        PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(FeatureTest.class);
+        PainlessClass s = painlessLookup.lookupPainlessClass(FeatureTest.class);
         PainlessMethod getter = s.methods.get(PainlessLookupUtility.buildPainlessMethodKey("getX", 0));
         PainlessMethod setter = s.methods.get(PainlessLookupUtility.buildPainlessMethodKey("setX", 1));
         PSubShortcut node = new PSubShortcut(l, "x", FeatureTest.class.getName(), getter, setter);
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml
index 20047e7d4825d..f2e1cb616b980 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml
@@ -132,7 +132,7 @@
           body:
             script:
               lang:   painless
-              source: "for (def key : params.keySet()) { ctx._source[key] = params[key]}"
+              source: "ctx._source.ctx = ctx"
               params: { bar: 'xxx' }
 
   - match: { error.root_cause.0.type: "remote_transport_exception" }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml
index 2914e8a916ec6..3be6601521e87 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml
@@ -108,7 +108,7 @@ setup:
                 script_fields:
                     bar:
                         script:
-                            source: "doc.date.value.dayOfWeek"
+                            source: "doc.date.value.dayOfWeek.value"
 
     - match: { hits.hits.0.fields.bar.0: 7}
 
@@ -123,7 +123,7 @@ setup:
                             source: >
                               StringBuilder b = new StringBuilder();
                               for (def date : doc.dates) {
-                                b.append(" ").append(date.getDayOfWeek());
+                                b.append(" ").append(date.getDayOfWeek().value);
                               }
                               return b.toString().trim()
 
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml
index 617b8df61b6bd..4c3c204d2d9ca 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml
@@ -95,7 +95,7 @@ setup:
                     field:
                         script:
                             source: "doc.date.get(0)"
-    - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' }
+    - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12Z' }
 
     - do:
         search:
@@ -104,7 +104,7 @@ setup:
                     field:
                         script:
                             source: "doc.date.value"
-    - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' }
+    - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12Z' }
 
 ---
 "geo_point":
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
index f764364380fcf..546677a2be4f4 100644
--- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
@@ -88,9 +88,9 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     }
 
     @Override
-    protected Settings indexSettings() {
+    protected Settings createTestIndexSettings() {
         return Settings.builder()
-            .put(super.indexSettings())
+            .put(super.createTestIndexSettings())
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .build();
     }
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java
index 63d94f39b2231..6d6822007eee3 100644
--- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java
@@ -73,9 +73,9 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     }
 
     @Override
-    protected Settings indexSettings() {
+    protected Settings createTestIndexSettings() {
         return Settings.builder()
-            .put(super.indexSettings())
+            .put(super.createTestIndexSettings())
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .build();
     }
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java
index 5a128f4d305fb..83441ef92d2b4 100644
--- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java
@@ -62,9 +62,9 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     }
 
     @Override
-    protected Settings indexSettings() {
+    protected Settings createTestIndexSettings() {
         return Settings.builder()
-            .put(super.indexSettings())
+            .put(super.createTestIndexSettings())
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .build();
     }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index e28fbead29abd..f18efe4585bc9 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -756,7 +756,7 @@ public BitSetProducer bitsetFilter(Query query) {
             @Override
             @SuppressWarnings("unchecked")
             public <IFD extends IndexFieldData<?>> IFD getForField(MappedFieldType fieldType) {
-                IndexFieldData.Builder builder = fieldType.fielddataBuilder(shardContext.getFullyQualifiedIndexName());
+                IndexFieldData.Builder builder = fieldType.fielddataBuilder(shardContext.getFullyQualifiedIndex().getName());
                 IndexFieldDataCache cache = new IndexFieldDataCache.None();
                 CircuitBreakerService circuitBreaker = new NoneCircuitBreakerService();
                 return (IFD) builder.build(shardContext.getIndexSettings(), fieldType, cache, circuitBreaker,
@@ -764,5 +764,4 @@ public <IFD extends IndexFieldData<?>> IFD getForField(MappedFieldType fieldType
             }
         };
     }
-
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index 5bd83b6c19a2e..731a27aa72c84 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -48,9 +48,9 @@
 import org.elasticsearch.index.mapper.TypeFieldMapper;
 import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
-import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.UpdateScript;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -746,7 +746,7 @@ public abstract static class ScriptApplier implements BiFunction<RequestWrapper<
         private final Script script;
         private final Map<String, Object> params;
 
-        private ExecutableScript executable;
+        private UpdateScript executable;
         private Map<String, Object> context;
 
         public ScriptApplier(WorkerBulkByScrollTaskState taskWorker,
@@ -766,7 +766,7 @@ public RequestWrapper<?> apply(RequestWrapper<?> request, ScrollableHitSource.Hi
                 return request;
             }
             if (executable == null) {
-                ExecutableScript.Factory factory = scriptService.compile(script, ExecutableScript.UPDATE_CONTEXT);
+                UpdateScript.Factory factory = scriptService.compile(script, UpdateScript.CONTEXT);
                 executable = factory.newInstance(params);
             }
             if (context == null) {
@@ -787,8 +787,7 @@ public RequestWrapper<?> apply(RequestWrapper<?> request, ScrollableHitSource.Hi
             OpType oldOpType = OpType.INDEX;
             context.put("op", oldOpType.toString());
 
-            executable.setNextVar("ctx", context);
-            executable.run();
+            executable.execute(context);
 
             String newOp = (String) context.remove("op");
             if (newOp == null) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
index c1defe56adc6f..706f2c0b8f8f1 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
@@ -27,13 +27,13 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.function.Supplier;
 
 public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteByQueryRequest, BulkByScrollResponse> {
 
@@ -46,7 +46,7 @@ public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteB
     public TransportDeleteByQueryAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, Client client,
                                         TransportService transportService, ScriptService scriptService, ClusterService clusterService) {
         super(settings, DeleteByQueryAction.NAME, transportService, actionFilters,
-            (Supplier<DeleteByQueryRequest>) DeleteByQueryRequest::new);
+            (Writeable.Reader<DeleteByQueryRequest>) DeleteByQueryRequest::new);
         this.threadPool = threadPool;
         this.client = client;
         this.scriptService = scriptService;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
index e54b5f50ae674..1d80c28b58568 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.action.index.IndexRequest;
@@ -104,7 +105,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
     public TransportReindexAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
             IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, ScriptService scriptService,
             AutoCreateIndex autoCreateIndex, Client client, TransportService transportService) {
-        super(settings, ReindexAction.NAME, transportService, actionFilters, ReindexRequest::new);
+        super(settings, ReindexAction.NAME, transportService, actionFilters, (Writeable.Reader<ReindexRequest>)ReindexRequest::new);
         this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.scriptService = scriptService;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
index 34ae3fdd0c62f..00d14822ba091 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
@@ -43,7 +44,6 @@
 
 import java.util.Map;
 import java.util.function.BiFunction;
-import java.util.function.Supplier;
 
 public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateByQueryRequest, BulkByScrollResponse> {
 
@@ -56,7 +56,7 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
     public TransportUpdateByQueryAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, Client client,
                                         TransportService transportService, ScriptService scriptService, ClusterService clusterService) {
         super(settings, UpdateByQueryAction.NAME, transportService, actionFilters,
-            (Supplier<UpdateByQueryRequest>) UpdateByQueryRequest::new);
+            (Writeable.Reader<UpdateByQueryRequest>) UpdateByQueryRequest::new);
         this.threadPool = threadPool;
         this.client = client;
         this.scriptService = scriptService;
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java
index 955c99568c7d2..94f375e9333ed 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java
@@ -26,8 +26,10 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.UpdateScript;
 import org.junit.Before;
 
+import java.util.Collections;
 import java.util.Map;
 import java.util.function.Consumer;
 
@@ -54,10 +56,16 @@ public void setupScriptService() {
     protected <T extends ActionRequest> T applyScript(Consumer<Map<String, Object>> scriptBody) {
         IndexRequest index = new IndexRequest("index", "type", "1").source(singletonMap("foo", "bar"));
         ScrollableHitSource.Hit doc = new ScrollableHitSource.BasicHit("test", "type", "id", 0);
-        ExecutableScript executableScript = new SimpleExecutableScript(scriptBody);
-        ExecutableScript.Factory factory = params -> executableScript;
-        when(scriptService.compile(any(), eq(ExecutableScript.CONTEXT))).thenReturn(factory);
-        when(scriptService.compile(any(), eq(ExecutableScript.UPDATE_CONTEXT))).thenReturn(factory);
+        UpdateScript updateScript = new UpdateScript(Collections.emptyMap()) {
+            @Override
+            public void execute(Map<String, Object> ctx) {
+                scriptBody.accept(ctx);
+            }
+        };
+        UpdateScript.Factory factory = params -> updateScript;
+        ExecutableScript simpleExecutableScript = new SimpleExecutableScript(scriptBody);
+        when(scriptService.compile(any(), eq(ExecutableScript.CONTEXT))).thenReturn(params -> simpleExecutableScript);
+        when(scriptService.compile(any(), eq(UpdateScript.CONTEXT))).thenReturn(factory);
         AbstractAsyncBulkByScrollAction<Request> action = action(scriptService, request().setScript(mockScript("")));
         RequestWrapper<?> result = action.buildScriptApplier().apply(AbstractAsyncBulkByScrollAction.wrap(index), doc);
         return (result != null) ? (T) result.self() : null;
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
index 2dc4b59e8d9f9..97809c9bc8dc3 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
@@ -67,19 +67,17 @@ public void testReindexRequest() throws IOException {
                 new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), port, null,
                     query, username, password, headers, socketTimeout, connectTimeout));
         }
-        ReindexRequest tripped = new ReindexRequest();
-        roundTrip(reindex, tripped);
+        ReindexRequest tripped = new ReindexRequest(toInputByteStream(reindex));
         assertRequestEquals(reindex, tripped);
 
         // Try slices=auto with a version that doesn't support it, which should fail
         reindex.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
-        Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_6_0_0_alpha1, reindex, null));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> toInputByteStream(Version.V_6_0_0_alpha1, reindex));
         assertEquals("Slices set as \"auto\" are not supported before version [6.1.0]. Found version [6.0.0-alpha1]", e.getMessage());
 
         // Try regular slices with a version that doesn't support slices=auto, which should succeed
-        tripped = new ReindexRequest();
         reindex.setSlices(between(1, Integer.MAX_VALUE));
-        roundTrip(Version.V_6_0_0_alpha1, reindex, tripped);
+        tripped = new ReindexRequest(toInputByteStream(reindex));
         assertRequestEquals(Version.V_6_0_0_alpha1, reindex, tripped);
     }
 
@@ -89,20 +87,18 @@ public void testUpdateByQueryRequest() throws IOException {
         if (randomBoolean()) {
             update.setPipeline(randomAlphaOfLength(5));
         }
-        UpdateByQueryRequest tripped = new UpdateByQueryRequest();
-        roundTrip(update, tripped);
+        UpdateByQueryRequest tripped = new UpdateByQueryRequest(toInputByteStream(update));
         assertRequestEquals(update, tripped);
         assertEquals(update.getPipeline(), tripped.getPipeline());
 
         // Try slices=auto with a version that doesn't support it, which should fail
         update.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
-        Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_6_0_0_alpha1, update, null));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> toInputByteStream(Version.V_6_0_0_alpha1, update));
         assertEquals("Slices set as \"auto\" are not supported before version [6.1.0]. Found version [6.0.0-alpha1]", e.getMessage());
 
         // Try regular slices with a version that doesn't support slices=auto, which should succeed
-        tripped = new UpdateByQueryRequest();
         update.setSlices(between(1, Integer.MAX_VALUE));
-        roundTrip(Version.V_6_0_0_alpha1, update, tripped);
+        tripped = new UpdateByQueryRequest(toInputByteStream(update));
         assertRequestEquals(update, tripped);
         assertEquals(update.getPipeline(), tripped.getPipeline());
     }
@@ -110,19 +106,17 @@ public void testUpdateByQueryRequest() throws IOException {
     public void testDeleteByQueryRequest() throws IOException {
         DeleteByQueryRequest delete = new DeleteByQueryRequest(new SearchRequest());
         randomRequest(delete);
-        DeleteByQueryRequest tripped = new DeleteByQueryRequest();
-        roundTrip(delete, tripped);
+        DeleteByQueryRequest tripped = new DeleteByQueryRequest(toInputByteStream(delete));
         assertRequestEquals(delete, tripped);
 
         // Try slices=auto with a version that doesn't support it, which should fail
         delete.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
-        Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_6_0_0_alpha1, delete, null));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> toInputByteStream(Version.V_6_0_0_alpha1, delete));
         assertEquals("Slices set as \"auto\" are not supported before version [6.1.0]. Found version [6.0.0-alpha1]", e.getMessage());
 
         // Try regular slices with a version that doesn't support slices=auto, which should succeed
-        tripped = new DeleteByQueryRequest();
         delete.setSlices(between(1, Integer.MAX_VALUE));
-        roundTrip(Version.V_6_0_0_alpha1, delete, tripped);
+        tripped = new DeleteByQueryRequest(toInputByteStream(delete));
         assertRequestEquals(delete, tripped);
     }
 
@@ -198,23 +192,24 @@ public void testRethrottleRequest() throws IOException {
             request.setTaskId(new TaskId(randomAlphaOfLength(5), randomLong()));
         }
         RethrottleRequest tripped = new RethrottleRequest();
-        roundTrip(request, tripped);
+        // We use readFrom here because Rethrottle does not support the Writeable.Reader interface
+        tripped.readFrom(toInputByteStream(request));
         assertEquals(request.getRequestsPerSecond(), tripped.getRequestsPerSecond(), 0.00001);
         assertArrayEquals(request.getActions(), tripped.getActions());
         assertEquals(request.getTaskId(), tripped.getTaskId());
     }
 
-    private void roundTrip(Streamable example, Streamable empty) throws IOException {
-        roundTrip(Version.CURRENT, example, empty);
+    private StreamInput toInputByteStream(Streamable example) throws IOException {
+        return toInputByteStream(Version.CURRENT, example);
     }
 
-    private void roundTrip(Version version, Streamable example, Streamable empty) throws IOException {
+    private StreamInput toInputByteStream(Version version, Streamable example) throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         out.setVersion(version);
         example.writeTo(out);
         StreamInput in = out.bytes().streamInput();
         in.setVersion(version);
-        empty.readFrom(in);
+        return in;
     }
 
     private Script randomScript() {
diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
index 5d4bcd7c10a84..12ce5ce7d4a8f 100644
--- a/modules/transport-netty4/build.gradle
+++ b/modules/transport-netty4/build.gradle
@@ -34,13 +34,13 @@ compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-tr
 
 dependencies {
     // network stack
-    compile "io.netty:netty-buffer:4.1.16.Final"
-    compile "io.netty:netty-codec:4.1.16.Final"
-    compile "io.netty:netty-codec-http:4.1.16.Final"
-    compile "io.netty:netty-common:4.1.16.Final"
-    compile "io.netty:netty-handler:4.1.16.Final"
-    compile "io.netty:netty-resolver:4.1.16.Final"
-    compile "io.netty:netty-transport:4.1.16.Final"
+    compile "io.netty:netty-buffer:4.1.28.Final"
+    compile "io.netty:netty-codec:4.1.28.Final"
+    compile "io.netty:netty-codec-http:4.1.28.Final"
+    compile "io.netty:netty-common:4.1.28.Final"
+    compile "io.netty:netty-handler:4.1.28.Final"
+    compile "io.netty:netty-resolver:4.1.28.Final"
+    compile "io.netty:netty-transport:4.1.28.Final"
 }
 
 dependencyLicenses {
@@ -134,7 +134,6 @@ thirdPartyAudit.excludes = [
         'net.jpountz.xxhash.StreamingXXHash32',
         'net.jpountz.xxhash.XXHashFactory',
         'io.netty.internal.tcnative.CertificateRequestedCallback',
-        'io.netty.internal.tcnative.CertificateRequestedCallback$KeyMaterial',
         'io.netty.internal.tcnative.CertificateVerifier',
         'io.netty.internal.tcnative.SessionTicketKey',
         'io.netty.internal.tcnative.SniHostNameMatcher',
@@ -161,6 +160,6 @@ thirdPartyAudit.excludes = [
 
         'org.conscrypt.AllocatedBuffer',
         'org.conscrypt.BufferAllocator',
-        'org.conscrypt.Conscrypt$Engines',
+        'org.conscrypt.Conscrypt',
         'org.conscrypt.HandshakeListener'
 ]
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.16.Final.jar.sha1
deleted file mode 100644
index c546222971985..0000000000000
--- a/modules/transport-netty4/licenses/netty-buffer-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-63b5fa95c74785e16f2c30ce268bc222e35c8cb5
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..f8a652d0dd1b4
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-buffer-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+d6c2d13492778009d33f60e05ed90bcb535d1fd1
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.16.Final.jar.sha1
deleted file mode 100644
index 1e6c241ea0b17..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d84a1f21768b7309c2954521cf5a1f46c2309eb1
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..70799bf10328a
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+a38361d893900947524f8a9da980555950e73d6a
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.16.Final.jar.sha1
deleted file mode 100644
index 71c33af1c5fc2..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-http-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d64312378b438dfdad84267c599a053327c6f02a
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..e1d34ebf89bb8
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+897100c1022c780b0a436b9349e507e8fa9800dc
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.16.Final.jar.sha1
deleted file mode 100644
index 3edf5fcea59b3..0000000000000
--- a/modules/transport-netty4/licenses/netty-common-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-177a6b30cca92f6f5f9873c9befd681377a4c328
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..bc95142683242
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-common-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+df69ce8bb9b544a71e7bbee290253cf7c93e6bad
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.16.Final.jar.sha1
deleted file mode 100644
index cba27387268d1..0000000000000
--- a/modules/transport-netty4/licenses/netty-handler-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fec0e63e7dd7f4eeef7ea8dc47a1ff32dfc7ebc2
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..80dc8b8f6fe95
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-handler-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+a035784682da0126bc25f10713dac732b5082a6d
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.16.Final.jar.sha1
deleted file mode 100644
index 3571d2ecfdc48..0000000000000
--- a/modules/transport-netty4/licenses/netty-resolver-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f6eb553b53fb3a90a8ac1170697093fed82eae28
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..afe004bd71623
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-resolver-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+f33557dcb31fa20da075ac05e4808115e32ef9b7
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.16.Final.jar.sha1
deleted file mode 100644
index e502d4c77084c..0000000000000
--- a/modules/transport-netty4/licenses/netty-transport-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3c8ee2c4d4a1cbb947a5c184c7aeb2204260958b
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..af19a16d6ed64
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-transport-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+d2ef28f49d726737f0ffe84bf66529b3bf6e0c0d
\ No newline at end of file
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
index c8c6fceb54304..90138bc59906e 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
@@ -267,8 +267,12 @@ protected Netty4TcpServerChannel bind(String name, InetSocketAddress address) {
         return esChannel;
     }
 
-    ScheduledPing getPing() {
-        return scheduledPing;
+    long successfulPingCount() {
+        return successfulPings.count();
+    }
+
+    long failedPingCount() {
+        return failedPings.count();
     }
 
     @Override
diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
index 32b2dc9bd1540..b8099025e47bf 100644
--- a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
+++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
@@ -23,6 +23,9 @@ grant codeBase "${codebase.netty-common}" {
 
    // netty makes and accepts socket connections
    permission java.net.SocketPermission "*", "accept,connect";
+
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };
 
 grant codeBase "${codebase.netty-transport}" {
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java
index 094f339059876..a005ae1054662 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -92,15 +93,19 @@ public void dispatchBadRequest(RestRequest request, RestChannel channel, ThreadC
             try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
                 final Collection<FullHttpResponse> responses =
                         nettyHttpClient.get(transportAddress.address(), "/_cluster/settings?pretty=%");
-                assertThat(responses, hasSize(1));
-                assertThat(responses.iterator().next().status().code(), equalTo(400));
-                final Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
-                assertThat(responseBodies, hasSize(1));
-                assertThat(responseBodies.iterator().next(), containsString("\"type\":\"bad_parameter_exception\""));
-                assertThat(
+                try {
+                    assertThat(responses, hasSize(1));
+                    assertThat(responses.iterator().next().status().code(), equalTo(400));
+                    final Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
+                    assertThat(responseBodies, hasSize(1));
+                    assertThat(responseBodies.iterator().next(), containsString("\"type\":\"bad_parameter_exception\""));
+                    assertThat(
                         responseBodies.iterator().next(),
                         containsString(
-                                "\"reason\":\"java.lang.IllegalArgumentException: unterminated escape sequence at end of string: %\""));
+                            "\"reason\":\"java.lang.IllegalArgumentException: unterminated escape sequence at end of string: %\""));
+                } finally {
+                    responses.forEach(ReferenceCounted::release);
+                }
             }
         }
     }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
index 0fa331ba138f6..a595de3a47ed9 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
@@ -58,6 +58,7 @@
 
 import static io.netty.handler.codec.http.HttpHeaderNames.HOST;
 import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+import static org.junit.Assert.fail;
 
 /**
  * Tiny helper to send http requests over netty.
@@ -145,7 +146,9 @@ private synchronized Collection<FullHttpResponse> sendRequests(
             for (HttpRequest request : requests) {
                 channelFuture.channel().writeAndFlush(request);
             }
-            latch.await(30, TimeUnit.SECONDS);
+            if (latch.await(30L, TimeUnit.SECONDS) == false) {
+                fail("Failed to get all expected responses.");
+            }
 
         } finally {
             if (channelFuture != null) {
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
index 269773fbb634c..ad81f5b3063f5 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.ESNetty4IntegTestCase;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
@@ -88,12 +89,20 @@ public void testLimitsInFlightRequests() throws Exception {
 
         try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
             Collection<FullHttpResponse> singleResponse = nettyHttpClient.post(transportAddress.address(), requests[0]);
-            assertThat(singleResponse, hasSize(1));
-            assertAtLeastOnceExpectedStatus(singleResponse, HttpResponseStatus.OK);
-
-            Collection<FullHttpResponse> multipleResponses = nettyHttpClient.post(transportAddress.address(), requests);
-            assertThat(multipleResponses, hasSize(requests.length));
-            assertAtLeastOnceExpectedStatus(multipleResponses, HttpResponseStatus.SERVICE_UNAVAILABLE);
+            try {
+                assertThat(singleResponse, hasSize(1));
+                assertAtLeastOnceExpectedStatus(singleResponse, HttpResponseStatus.OK);
+
+                Collection<FullHttpResponse> multipleResponses = nettyHttpClient.post(transportAddress.address(), requests);
+                try {
+                    assertThat(multipleResponses, hasSize(requests.length));
+                    assertAtLeastOnceExpectedStatus(multipleResponses, HttpResponseStatus.SERVICE_UNAVAILABLE);
+                } finally {
+                    multipleResponses.forEach(ReferenceCounted::release);
+                }
+            } finally {
+                singleResponse.forEach(ReferenceCounted::release);
+            }
         }
     }
 
@@ -113,8 +122,12 @@ public void testDoesNotLimitExcludedRequests() throws Exception {
 
         try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
             Collection<FullHttpResponse> responses = nettyHttpClient.put(transportAddress.address(), requestUris);
-            assertThat(responses, hasSize(requestUris.length));
-            assertAllInExpectedStatus(responses, HttpResponseStatus.OK);
+            try {
+                assertThat(responses, hasSize(requestUris.length));
+                assertAllInExpectedStatus(responses, HttpResponseStatus.OK);
+            } finally {
+                responses.forEach(ReferenceCounted::release);
+            }
         }
     }
 
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
index 50be7f7ce4509..cb4d14038d24a 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
@@ -29,6 +29,7 @@
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.FullHttpResponse;
 import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -98,8 +99,12 @@ public void testThatHttpPipeliningWorks() throws Exception {
 
             try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
                 Collection<FullHttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{}));
-                Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
-                assertThat(responseBodies, contains(requests.toArray()));
+                try {
+                    Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
+                    assertThat(responseBodies, contains(requests.toArray()));
+                } finally {
+                    responses.forEach(ReferenceCounted::release);
+                }
             }
         }
     }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
index bcf28506143bf..1c3c71d710d17 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
@@ -207,14 +207,23 @@ public void dispatchBadRequest(RestRequest request, RestChannel channel, ThreadC
                 HttpUtil.setContentLength(request, contentLength);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(expectedStatus));
-                if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
-                    final FullHttpRequest continuationRequest =
-                        new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
-                    final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
-
-                    assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
-                    assertThat(new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done"));
+                try {
+                    assertThat(response.status(), equalTo(expectedStatus));
+                    if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
+                        final FullHttpRequest continuationRequest =
+                            new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
+                        final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
+                        try {
+                            assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
+                            assertThat(
+                                new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done")
+                            );
+                        } finally {
+                            continuationResponse.release();
+                        }
+                    }
+                } finally {
+                    response.release();
                 }
             }
         }
@@ -280,10 +289,14 @@ public void dispatchBadRequest(final RestRequest request,
                 final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
-                assertThat(
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
+                    assertThat(
                         new String(response.content().array(), Charset.forName("UTF-8")),
                         containsString("you sent a bad request and you should feel bad"));
+                } finally {
+                    response.release();
+                }
             }
         }
 
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java
index ebb91d9663ed5..275442cd50aa0 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.ESNetty4IntegTestCase;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.http.HttpServerTransport;
@@ -45,14 +46,18 @@ public void testThatNettyHttpServerSupportsPipelining() throws Exception {
 
         HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
         TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses();
-        TransportAddress transportAddress = (TransportAddress) randomFrom(boundAddresses);
+        TransportAddress transportAddress = randomFrom(boundAddresses);
 
         try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
             Collection<FullHttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests);
-            assertThat(responses, hasSize(5));
+            try {
+                assertThat(responses, hasSize(5));
 
-            Collection<String> opaqueIds = Netty4HttpClient.returnOpaqueIds(responses);
-            assertOpaqueIdsInOrder(opaqueIds);
+                Collection<String> opaqueIds = Netty4HttpClient.returnOpaqueIds(responses);
+                assertOpaqueIdsInOrder(opaqueIds);
+            } finally {
+                responses.forEach(ReferenceCounted::release);
+            }
         }
     }
 
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java
index 01c5f5b617077..fae4082e81828 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java
@@ -26,16 +26,13 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TcpTransport;
-import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseHandler;
@@ -83,22 +80,19 @@ public void testScheduledPing() throws Exception {
         serviceB.connectToNode(nodeA);
 
         assertBusy(() -> {
-            assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(100L));
-            assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(100L));
+            assertThat(nettyA.successfulPingCount(), greaterThan(100L));
+            assertThat(nettyB.successfulPingCount(), greaterThan(100L));
         });
-        assertThat(nettyA.getPing().getFailedPings(), equalTo(0L));
-        assertThat(nettyB.getPing().getFailedPings(), equalTo(0L));
+        assertThat(nettyA.failedPingCount(), equalTo(0L));
+        assertThat(nettyB.failedPingCount(), equalTo(0L));
 
         serviceA.registerRequestHandler("internal:sayHello", TransportRequest.Empty::new, ThreadPool.Names.GENERIC,
-            new TransportRequestHandler<TransportRequest.Empty>() {
-                @Override
-                public void messageReceived(TransportRequest.Empty request, TransportChannel channel, Task task) {
-                    try {
-                        channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.EMPTY);
-                    } catch (IOException e) {
-                        logger.error("Unexpected failure", e);
-                        fail(e.getMessage());
-                    }
+            (request, channel, task) -> {
+                try {
+                    channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.EMPTY);
+                } catch (IOException e) {
+                    logger.error("Unexpected failure", e);
+                    fail(e.getMessage());
                 }
             });
 
@@ -130,11 +124,11 @@ public void handleException(TransportException exp) {
         }
 
         assertBusy(() -> {
-            assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(200L));
-            assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(200L));
+            assertThat(nettyA.successfulPingCount(), greaterThan(200L));
+            assertThat(nettyB.successfulPingCount(), greaterThan(200L));
         });
-        assertThat(nettyA.getPing().getFailedPings(), equalTo(0L));
-        assertThat(nettyB.getPing().getFailedPings(), equalTo(0L));
+        assertThat(nettyA.failedPingCount(), equalTo(0L));
+        assertThat(nettyB.failedPingCount(), equalTo(0L));
 
         Releasables.close(serviceA, serviceB);
         terminate(threadPool);
diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle
index ad5a7b7c57b61..1883e3bf1b9d6 100644
--- a/plugins/analysis-icu/build.gradle
+++ b/plugins/analysis-icu/build.gradle
@@ -30,9 +30,9 @@ forbiddenApis {
 
 dependencies {
   compile "org.apache.lucene:lucene-analyzers-icu:${versions.lucene}"
-  compile 'com.ibm.icu:icu4j:62.1'
+  compile "com.ibm.icu:icu4j:${versions.icu4j}"
 }
 
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
-}
\ No newline at end of file
+}
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..1e79e1e70ef8f
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+a010e852be8d56efe1906e6da5292e4541239724
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index 5b6947a9c7578..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7a37816def72a748416c4ae8b0f6817e30efb99f
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..2d9669e436229
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+88e0ed90d433a9088528485cd4f59311735d92a4
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index d39638c188466..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ca7437178cdbf7b8bfe0d75c75e3c8eb93925724
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..f7b8fdd4bc187
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+0daec9ac3c4bba5f91b1bc413c651b7a98313982
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index 21c25d2bb2404..0000000000000
--- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3f5dec44f380d6d58bc1c8aec51964fcb5390b60
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..80cf627011b4e
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+f5af81eec04c1da0d6969cff18f360ff379b1bf7
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index f58c597eadd6d..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-453bf1d60df0415439095624e0b3e42492ad4716
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..14be684b96f3d
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+9e649088ee298293aa95a05391dff9cb0582648e
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index 8ccec8dbf3786..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-70095a45257bca9f46629b5fb6cedf9eff5e2b07
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..ea55c790537f4
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+47fb370054ba7413d050f13c177edf01180c31ca
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index ec9c33119f556..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7199d6962d268b7877f7b5160e98e4ff21cce5c7
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..2d6f580c35a23
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+bc0708acbac195772b67b5ad2e9c4683d27ff450
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index ba9148ef1b32a..0000000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-12aff508d39d206a1aead5013ecd11882062eb06
\ No newline at end of file
diff --git a/plugins/examples/custom-suggester/build.gradle b/plugins/examples/custom-suggester/build.gradle
new file mode 100644
index 0000000000000..b36d5cd218d27
--- /dev/null
+++ b/plugins/examples/custom-suggester/build.gradle
@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+    name 'custom-suggester'
+    description 'An example plugin showing how to write and register a custom suggester'
+    classname 'org.elasticsearch.example.customsuggester.CustomSuggesterPlugin'
+}
+
+integTestCluster {
+    numNodes = 2
+}
+
+// this plugin has no unit tests, only rest tests
+tasks.test.enabled = false
\ No newline at end of file
diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java
new file mode 100644
index 0000000000000..b6a5b5e8f842f
--- /dev/null
+++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.customsuggester;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.util.CharsRefBuilder;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.Suggester;
+
+import java.util.Locale;
+
+public class CustomSuggester extends Suggester<CustomSuggestionContext> {
+
+    // This is a pretty dumb implementation which returns the original text + fieldName + custom config option + 12 or 123
+    @Override
+    public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> innerExecute(
+        String name,
+        CustomSuggestionContext suggestion,
+        IndexSearcher searcher,
+        CharsRefBuilder spare) {
+
+        // Get the suggestion context
+        String text = suggestion.getText().utf8ToString();
+
+        // create two suggestions with 12 and 123 appended
+        CustomSuggestion response = new CustomSuggestion(name, suggestion.getSize(), "suggestion-dummy-value");
+
+        CustomSuggestion.Entry entry = new CustomSuggestion.Entry(new Text(text), 0, text.length(), "entry-dummy-value");
+
+        String firstOption =
+            String.format(Locale.ROOT, "%s-%s-%s-%s", text, suggestion.getField(), suggestion.options.get("suffix"), "12");
+        CustomSuggestion.Entry.Option option12 = new CustomSuggestion.Entry.Option(new Text(firstOption), 0.9f, "option-dummy-value-1");
+        entry.addOption(option12);
+
+        String secondOption =
+            String.format(Locale.ROOT, "%s-%s-%s-%s", text, suggestion.getField(), suggestion.options.get("suffix"), "123");
+        CustomSuggestion.Entry.Option option123 = new CustomSuggestion.Entry.Option(new Text(secondOption), 0.8f, "option-dummy-value-2");
+        entry.addOption(option123);
+
+        response.addTerm(entry);
+
+        return response;
+    }
+}
diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggesterPlugin.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggesterPlugin.java
new file mode 100644
index 0000000000000..91ffa672e5351
--- /dev/null
+++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggesterPlugin.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.customsuggester;
+
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+
+import java.util.Collections;
+import java.util.List;
+
+public class CustomSuggesterPlugin extends Plugin implements SearchPlugin {
+    @Override
+    public List<SearchPlugin.SuggesterSpec<?>> getSuggesters() {
+        return Collections.singletonList(
+            new SearchPlugin.SuggesterSpec<>(
+                CustomSuggestionBuilder.SUGGESTION_NAME,
+                CustomSuggestionBuilder::new,
+                CustomSuggestionBuilder::fromXContent,
+                CustomSuggestion::new
+            )
+        );
+    }
+}
diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java
new file mode 100644
index 0000000000000..f7ec27b7af002
--- /dev/null
+++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.customsuggester;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.suggest.Suggest;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class CustomSuggestion extends Suggest.Suggestion<CustomSuggestion.Entry> {
+
+    public static final int TYPE = 999;
+
+    public static final ParseField DUMMY = new ParseField("dummy");
+
+    private String dummy;
+
+    public CustomSuggestion(String name, int size, String dummy) {
+        super(name, size);
+        this.dummy = dummy;
+    }
+
+    public CustomSuggestion(StreamInput in) throws IOException {
+        super(in);
+        dummy = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(dummy);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return CustomSuggestionBuilder.SUGGESTION_NAME;
+    }
+
+    @Override
+    public int getWriteableType() {
+        return TYPE;
+    }
+
+    /**
+     * A meaningless value used to test that plugin suggesters can add fields to their Suggestion types
+     *
+     * This can't be serialized to xcontent because Suggestions appear in xcontent as an array of entries, so there is no place
+     * to add a custom field. But we can still use a custom field internally and use it to define a Suggestion's behavior
+     */
+    public String getDummy() {
+        return dummy;
+    }
+
+    @Override
+    protected Entry newEntry() {
+        return new Entry();
+    }
+
+    @Override
+    protected Entry newEntry(StreamInput in) throws IOException {
+        return new Entry(in);
+    }
+
+    public static CustomSuggestion fromXContent(XContentParser parser, String name) throws IOException {
+        CustomSuggestion suggestion = new CustomSuggestion(name, -1, null);
+        parseEntries(parser, suggestion, Entry::fromXContent);
+        return suggestion;
+    }
+
+    public static class Entry extends Suggest.Suggestion.Entry<CustomSuggestion.Entry.Option> {
+
+        private static final ObjectParser<Entry, Void> PARSER = new ObjectParser<>("CustomSuggestionEntryParser", true, Entry::new);
+
+        static {
+            declareCommonFields(PARSER);
+            PARSER.declareString((entry, dummy) -> entry.dummy = dummy, DUMMY);
+            PARSER.declareObjectArray(Entry::addOptions, (p, c) -> Option.fromXContent(p), new ParseField(OPTIONS));
+        }
+
+        private String dummy;
+
+        public Entry() {}
+
+        public Entry(Text text, int offset, int length, String dummy) {
+            super(text, offset, length);
+            this.dummy = dummy;
+        }
+
+        public Entry(StreamInput in) throws IOException {
+            super(in);
+            dummy = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(dummy);
+        }
+
+        @Override
+        protected Option newOption() {
+            return new Option();
+        }
+
+        @Override
+        protected Option newOption(StreamInput in) throws IOException {
+            return new Option(in);
+        }
+
+        /*
+         * the value of dummy will always be the same, so this just tests that we can merge entries with custom fields
+         */
+        @Override
+        protected void merge(Suggest.Suggestion.Entry<Option> otherEntry) {
+            dummy = ((Entry) otherEntry).getDummy();
+        }
+
+        /**
+         * Meaningless field used to test that plugin suggesters can add fields to their entries
+         */
+        public String getDummy() {
+            return dummy;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder = super.toXContent(builder, params);
+            builder.field(DUMMY.getPreferredName(), getDummy());
+            return builder;
+        }
+
+        public static Entry fromXContent(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        public static class Option extends Suggest.Suggestion.Entry.Option {
+
+            private static final ConstructingObjectParser<Option, Void> PARSER = new ConstructingObjectParser<>(
+                "CustomSuggestionObjectParser", true,
+                args -> {
+                    Text text = new Text((String) args[0]);
+                    float score = (float) args[1];
+                    String dummy = (String) args[2];
+                    return new Option(text, score, dummy);
+                });
+
+            static {
+                PARSER.declareString(constructorArg(), TEXT);
+                PARSER.declareFloat(constructorArg(), SCORE);
+                PARSER.declareString(constructorArg(), DUMMY);
+            }
+
+            private String dummy;
+
+            public Option() {}
+
+            public Option(Text text, float score, String dummy) {
+                super(text, score);
+                this.dummy = dummy;
+            }
+
+            public Option(StreamInput in) throws IOException {
+                super(in);
+                dummy = in.readString();
+            }
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+                super.writeTo(out);
+                out.writeString(dummy);
+            }
+
+            /**
+             * A meaningless value used to test that plugin suggesters can add fields to their options
+             */
+            public String getDummy() {
+                return dummy;
+            }
+
+            /*
+             * the value of dummy will always be the same, so this just tests that we can merge options with custom fields
+             */
+            @Override
+            protected void mergeInto(Suggest.Suggestion.Entry.Option otherOption) {
+                super.mergeInto(otherOption);
+                dummy = ((Option) otherOption).getDummy();
+            }
+
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                builder = super.toXContent(builder, params);
+                builder.field(DUMMY.getPreferredName(), dummy);
+                return builder;
+            }
+
+            public static Option fromXContent(XContentParser parser) {
+                return PARSER.apply(parser, null);
+            }
+        }
+    }
+}
diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java
new file mode 100644
index 0000000000000..9375062ab6918
--- /dev/null
+++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.customsuggester;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionSearchContext;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class CustomSuggestionBuilder extends SuggestionBuilder<CustomSuggestionBuilder> {
+
+    public static final String SUGGESTION_NAME = "custom";
+
+    protected static final ParseField RANDOM_SUFFIX_FIELD = new ParseField("suffix");
+
+    private String randomSuffix;
+
+    public CustomSuggestionBuilder(String randomField, String randomSuffix) {
+        super(randomField);
+        this.randomSuffix = randomSuffix;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public CustomSuggestionBuilder(StreamInput in) throws IOException {
+        super(in);
+        this.randomSuffix = in.readString();
+    }
+
+    @Override
+    public void doWriteTo(StreamOutput out) throws IOException {
+        out.writeString(randomSuffix);
+    }
+
+    @Override
+    protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(RANDOM_SUFFIX_FIELD.getPreferredName(), randomSuffix);
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return SUGGESTION_NAME;
+    }
+
+    @Override
+    protected boolean doEquals(CustomSuggestionBuilder other) {
+        return Objects.equals(randomSuffix, other.randomSuffix);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(randomSuffix);
+    }
+
+    public static CustomSuggestionBuilder fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token;
+        String currentFieldName = null;
+        String fieldname = null;
+        String suffix = null;
+        String analyzer = null;
+        int sizeField = -1;
+        int shardSize = -1;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (SuggestionBuilder.ANALYZER_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    analyzer = parser.text();
+                } else if (SuggestionBuilder.FIELDNAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    fieldname = parser.text();
+                } else if (SuggestionBuilder.SIZE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    sizeField = parser.intValue();
+                } else if (SuggestionBuilder.SHARDSIZE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    shardSize = parser.intValue();
+                } else if (RANDOM_SUFFIX_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    suffix = parser.text();
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(),
+                    "suggester[custom] doesn't support field [" + currentFieldName + "]");
+            }
+        }
+
+        // now we should have field name, check and copy fields over to the suggestion builder we return
+        if (fieldname == null) {
+            throw new ParsingException(parser.getTokenLocation(), "the required field option is missing");
+        }
+        CustomSuggestionBuilder builder = new CustomSuggestionBuilder(fieldname, suffix);
+        if (analyzer != null) {
+            builder.analyzer(analyzer);
+        }
+        if (sizeField != -1) {
+            builder.size(sizeField);
+        }
+        if (shardSize != -1) {
+            builder.shardSize(shardSize);
+        }
+        return builder;
+    }
+
+    @Override
+    public SuggestionSearchContext.SuggestionContext build(QueryShardContext context) throws IOException {
+        Map<String, Object> options = new HashMap<>();
+        options.put(FIELDNAME_FIELD.getPreferredName(), field());
+        options.put(RANDOM_SUFFIX_FIELD.getPreferredName(), randomSuffix);
+        CustomSuggestionContext customSuggestionsContext = new CustomSuggestionContext(context, options);
+        customSuggestionsContext.setField(field());
+        assert text != null;
+        customSuggestionsContext.setText(BytesRefs.toBytesRef(text));
+        return customSuggestionsContext;
+    }
+
+}
diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionContext.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionContext.java
new file mode 100644
index 0000000000000..a1d90993b226b
--- /dev/null
+++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionContext.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.customsuggester;
+
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.suggest.SuggestionSearchContext;
+
+import java.util.Map;
+
+public class CustomSuggestionContext extends SuggestionSearchContext.SuggestionContext {
+
+    public Map<String, Object> options;
+
+    public CustomSuggestionContext(QueryShardContext context, Map<String, Object> options) {
+        super(new CustomSuggester(), context);
+        this.options = options;
+    }
+}
diff --git a/plugins/examples/custom-suggester/src/test/java/org/elasticsearch/example/customsuggester/CustomSuggesterClientYamlTestSuiteIT.java b/plugins/examples/custom-suggester/src/test/java/org/elasticsearch/example/customsuggester/CustomSuggesterClientYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..b1282265f22a6
--- /dev/null
+++ b/plugins/examples/custom-suggester/src/test/java/org/elasticsearch/example/customsuggester/CustomSuggesterClientYamlTestSuiteIT.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.customsuggester;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class CustomSuggesterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public CustomSuggesterClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}
diff --git a/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/10_basic.yml b/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/10_basic.yml
new file mode 100644
index 0000000000000..29fbcdac99dd3
--- /dev/null
+++ b/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/10_basic.yml
@@ -0,0 +1,13 @@
+# tests that the custom suggester plugin is installed
+---
+"plugin loaded":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - contains:  { nodes.$master.plugins: { name: custom-suggester } }
diff --git a/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml b/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml
new file mode 100644
index 0000000000000..3731a8b211268
--- /dev/null
+++ b/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml
@@ -0,0 +1,55 @@
+# tests that the custom suggester works
+
+# the issue that prompted serializing Suggestion as a registered named writeable was not revealed until
+# a user found that it would fail when reducing suggestions in a multi node envrionment
+# https://github.com/elastic/elasticsearch/issues/26585
+"test custom suggester":
+  - do:
+      cluster.health:
+        wait_for_nodes: 2
+
+  - is_true: cluster_name
+  - is_false: timed_out
+  - gte: { number_of_nodes: 2 }
+  - gte: { number_of_data_nodes: 2 }
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 2
+            number_of_replicas: 0
+
+  - do:
+      bulk:
+        index: test
+        type: test
+        refresh: true
+        body: |
+          { "index": {} }
+          { "content": "these" }
+          { "index": {} }
+          { "content": "aren't" }
+          { "index": {} }
+          { "content": "actually" }
+          { "index": {} }
+          { "content": "used" }
+
+  - do:
+      search:
+        size: 0
+        index: test
+        body:
+          suggest:
+            test:
+              text: my suggestion text
+              custom:
+                field: arbitraryField
+                suffix: arbitrarySuffix
+
+  - match: { suggest.test.0.dummy: entry-dummy-value }
+  - match: { suggest.test.0.options.0.text: my suggestion text-arbitraryField-arbitrarySuffix-12 }
+  - match: { suggest.test.0.options.0.dummy: option-dummy-value-1 }
+  - match: { suggest.test.0.options.1.text: my suggestion text-arbitraryField-arbitrarySuffix-123 }
+  - match: { suggest.test.0.options.1.dummy: option-dummy-value-2 }
diff --git a/plugins/ingest-geoip/licenses/jackson-annotations-2.8.10.jar.sha1 b/plugins/ingest-geoip/licenses/jackson-annotations-2.8.10.jar.sha1
deleted file mode 100644
index 16addfe35076c..0000000000000
--- a/plugins/ingest-geoip/licenses/jackson-annotations-2.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5e924646d6f893bc9036939c5f2b4ecaee85e5da
\ No newline at end of file
diff --git a/plugins/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1 b/plugins/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..30e7d1a7b1a74
--- /dev/null
+++ b/plugins/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1
@@ -0,0 +1 @@
+391de20b4e29cb3fb07d2454ace64be2c82ac91f
\ No newline at end of file
diff --git a/plugins/ingest-geoip/licenses/jackson-databind-2.8.10.jar.sha1 b/plugins/ingest-geoip/licenses/jackson-databind-2.8.10.jar.sha1
deleted file mode 100644
index f8c5539424553..0000000000000
--- a/plugins/ingest-geoip/licenses/jackson-databind-2.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f7b83cb2bc4b88d53961e749e1ad32f49ef017b7
\ No newline at end of file
diff --git a/plugins/ingest-geoip/licenses/jackson-databind-2.8.11.jar.sha1 b/plugins/ingest-geoip/licenses/jackson-databind-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..53d6c1fa20834
--- /dev/null
+++ b/plugins/ingest-geoip/licenses/jackson-databind-2.8.11.jar.sha1
@@ -0,0 +1 @@
+0569a9f220273024523799dba9dd358121b0ee09
\ No newline at end of file
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 6d7412f8250e2..13ec8031c0fd8 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -23,8 +23,9 @@ esplugin {
 }
 
 dependencies {
-  compile 'com.microsoft.azure:azure-storage:5.0.0'
-  compile 'com.microsoft.azure:azure-keyvault-core:0.8.0'
+  compile 'com.microsoft.azure:azure-storage:8.0.0'
+  compile 'com.microsoft.azure:azure-keyvault-core:1.0.0'
+  compile 'com.google.guava:guava:20.0'
   compile 'org.apache.commons:commons-lang3:3.4'
 }
 
@@ -40,6 +41,18 @@ thirdPartyAudit.excludes = [
   // Optional and not enabled by Elasticsearch
   'org.slf4j.Logger',
   'org.slf4j.LoggerFactory',
+  // uses internal java api: sun.misc.Unsafe
+  'com.google.common.cache.Striped64',
+  'com.google.common.cache.Striped64$1',
+  'com.google.common.cache.Striped64$Cell',
+  'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+  'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+  'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
+  'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+  'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+  'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+  'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+  'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
 ]
 
 check {
diff --git a/plugins/repository-azure/licenses/azure-keyvault-core-0.8.0.jar.sha1 b/plugins/repository-azure/licenses/azure-keyvault-core-0.8.0.jar.sha1
deleted file mode 100644
index b86c58db8423a..0000000000000
--- a/plugins/repository-azure/licenses/azure-keyvault-core-0.8.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35f7ac687462f491d0f8b0d96733dfe347493d70
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-keyvault-core-1.0.0.jar.sha1 b/plugins/repository-azure/licenses/azure-keyvault-core-1.0.0.jar.sha1
new file mode 100644
index 0000000000000..8b44e99aee15d
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-keyvault-core-1.0.0.jar.sha1
@@ -0,0 +1 @@
+e89dd5e621e21b753096ec6a03f203c01482c612
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-storage-5.0.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-5.0.0.jar.sha1
deleted file mode 100644
index 9882cb80204ff..0000000000000
--- a/plugins/repository-azure/licenses/azure-storage-5.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ba8f04bfeac08016c0f88423a202d0f3aac03aed
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-storage-8.0.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-8.0.0.jar.sha1
new file mode 100644
index 0000000000000..4e333ad824184
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-storage-8.0.0.jar.sha1
@@ -0,0 +1 @@
+f6759c16ade4e2a05bc1dfbaf55161b9ed0e78b9
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/guava-20.0.jar.sha1 b/plugins/repository-azure/licenses/guava-20.0.jar.sha1
new file mode 100644
index 0000000000000..7b6ae09060b29
--- /dev/null
+++ b/plugins/repository-azure/licenses/guava-20.0.jar.sha1
@@ -0,0 +1 @@
+89507701249388e1ed5ddcf8c41f4ce1be7831ef
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/guava-LICENSE.txt b/plugins/repository-azure/licenses/guava-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/guava-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/guava-NOTICE.txt b/plugins/repository-azure/licenses/guava-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
index 9482182b02d28..75bcc3e9c1edb 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
@@ -61,7 +61,10 @@
 public class AzureStorageService extends AbstractComponent {
 
     public static final ByteSizeValue MIN_CHUNK_SIZE = new ByteSizeValue(1, ByteSizeUnit.BYTES);
-    public static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
+    /**
+     * {@link com.microsoft.azure.storage.blob.BlobConstants#MAX_SINGLE_UPLOAD_BLOB_SIZE_IN_BYTES}
+     */
+    public static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(256, ByteSizeUnit.MB);
 
     // 'package' for testing
     volatile Map<String, AzureStorageSettings> storageSettings = emptyMap();
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
index b4b71577cbcdc..43891a8e9d57c 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
@@ -104,7 +104,7 @@ public void testChunkSize() {
         assertEquals(AzureStorageService.MAX_CHUNK_SIZE, azureRepository.chunkSize());
 
         // chunk size in settings
-        int size = randomIntBetween(1, 64);
+        int size = randomIntBetween(1, 256);
         azureRepository = azureRepository(Settings.builder().put("chunk_size", size + "mb").build());
         assertEquals(new ByteSizeValue(size, ByteSizeUnit.MB), azureRepository.chunkSize());
 
@@ -120,8 +120,8 @@ public void testChunkSize() {
 
         // greater than max chunk size not allowed
         e = expectThrows(IllegalArgumentException.class, () ->
-            azureRepository(Settings.builder().put("chunk_size", "65mb").build()));
-        assertEquals("failed to parse value [65mb] for setting [chunk_size], must be <= [64mb]", e.getMessage());
+            azureRepository(Settings.builder().put("chunk_size", "257mb").build()));
+        assertEquals("failed to parse value [257mb] for setting [chunk_size], must be <= [256mb]", e.getMessage());
     }
 
 }
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index 8856ae1526a21..6debaf5282fc1 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -564,7 +564,6 @@ thirdPartyAudit.excludes = [
   // we are not pulling in slf4j-ext, this is okay, Log4j will fallback gracefully
   'org.slf4j.ext.EventData',
 
-  'org.apache.log4j.AppenderSkeleton',
   'org.apache.log4j.AsyncAppender',
   'org.apache.log4j.helpers.ISO8601DateFormat',
   'org.apache.log4j.spi.ThrowableInformation',
diff --git a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.11.1.jar.sha1 b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..6178556b31848
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.11.1.jar.sha1
@@ -0,0 +1 @@
+4b41b53a3a2d299ce381a69d165381ca19f62912
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.9.1.jar.sha1 b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.9.1.jar.sha1
deleted file mode 100644
index 66119e87e211f..0000000000000
--- a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.9.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0a97a849b18b3798c4af1a2ca5b10c66cef17e3a
\ No newline at end of file
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index 6001ed570652e..7f0ca209db797 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -132,7 +132,7 @@ if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3T
 final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z'
 final String minioBinDir = "${buildDir}/minio/bin"
 final String minioDataDir = "${buildDir}/minio/data"
-final String minioAddress = "127.0.0.1:60920"
+final String minioAddress = "127.0.0.1"
 
 final String minioDistribution
 final String minioCheckSum
@@ -187,15 +187,30 @@ if (useFixture && minioDistribution) {
     dependsOn installMinio
 
     ext.minioPid = 0L
+    ext.minioPort = 0
 
     doLast {
+      // get free port
+      for (int port = 60920; port < 60940; port++) {
+        try {
+          javax.net.ServerSocketFactory.getDefault().createServerSocket(port, 1, InetAddress.getByName(minioAddress)).close()
+          minioPort = port
+          break
+        } catch (BindException e) {
+          logger.info("Port " + port + " for Minio process is already taken", e)
+        }
+      }
+      if (minioPort == 0) {
+        throw new GradleException("Could not find a free port for Minio")
+      }
+
       new File("${minioDataDir}/${s3PermanentBucket}").mkdirs()
       // we skip these tests on Windows so we do no need to worry about compatibility here
       final ProcessBuilder minio = new ProcessBuilder(
               "${minioBinDir}/${minioFileName}",
               "server",
               "--address",
-              minioAddress,
+              minioAddress + ":" + minioPort,
               minioDataDir)
       minio.environment().put('MINIO_ACCESS_KEY', s3PermanentAccessKey)
       minio.environment().put('MINIO_SECRET_KEY', s3PermanentSecretKey)
@@ -227,6 +242,7 @@ if (useFixture && minioDistribution) {
             final int index = line.lastIndexOf(":")
             assert index >= 0
             httpPort = Integer.parseInt(line.substring(index + 1))
+            assert httpPort == minioPort : "Port mismatch, expected ${minioPort} but was ${httpPort}"
 
             final File script = new File(project.buildDir, "minio/minio.killer.sh")
             script.setText(
@@ -269,10 +285,15 @@ if (useFixture && minioDistribution) {
   project.afterEvaluate {
     ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration
     cluster.dependsOn(project.bundlePlugin)
+    cluster.dependsOn(startMinio) // otherwise we don't know the Minio port
     cluster.keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
     cluster.keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
 
-    cluster.setting 's3.client.integration_test_permanent.endpoint', "http://${minioAddress}"
+    Closure<String> minioAddressAndPort = {
+      assert startMinio.minioPort > 0
+      return 'http://' + minioAddress + ':' + startMinio.minioPort
+    }
+    cluster.setting 's3.client.integration_test_permanent.endpoint', "${ -> minioAddressAndPort.call()}"
 
     Task restIntegTestTask = project.tasks.getByName('integTestMinio')
     restIntegTestTask.clusterConfig.plugin(project.path)
diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle
index 856a8552b8e5a..07605bfee29b3 100644
--- a/plugins/transport-nio/build.gradle
+++ b/plugins/transport-nio/build.gradle
@@ -29,13 +29,13 @@ dependencies {
     compile "org.elasticsearch:elasticsearch-nio:${version}"
 
     // network stack
-    compile "io.netty:netty-buffer:4.1.16.Final"
-    compile "io.netty:netty-codec:4.1.16.Final"
-    compile "io.netty:netty-codec-http:4.1.16.Final"
-    compile "io.netty:netty-common:4.1.16.Final"
-    compile "io.netty:netty-handler:4.1.16.Final"
-    compile "io.netty:netty-resolver:4.1.16.Final"
-    compile "io.netty:netty-transport:4.1.16.Final"
+    compile "io.netty:netty-buffer:4.1.28.Final"
+    compile "io.netty:netty-codec:4.1.28.Final"
+    compile "io.netty:netty-codec-http:4.1.28.Final"
+    compile "io.netty:netty-common:4.1.28.Final"
+    compile "io.netty:netty-handler:4.1.28.Final"
+    compile "io.netty:netty-resolver:4.1.28.Final"
+    compile "io.netty:netty-transport:4.1.28.Final"
 }
 
 dependencyLicenses {
@@ -113,7 +113,6 @@ thirdPartyAudit.excludes = [
         'net.jpountz.xxhash.StreamingXXHash32',
         'net.jpountz.xxhash.XXHashFactory',
         'io.netty.internal.tcnative.CertificateRequestedCallback',
-        'io.netty.internal.tcnative.CertificateRequestedCallback$KeyMaterial',
         'io.netty.internal.tcnative.CertificateVerifier',
         'io.netty.internal.tcnative.SessionTicketKey',
         'io.netty.internal.tcnative.SniHostNameMatcher',
@@ -140,6 +139,6 @@ thirdPartyAudit.excludes = [
 
         'org.conscrypt.AllocatedBuffer',
         'org.conscrypt.BufferAllocator',
-        'org.conscrypt.Conscrypt$Engines',
+        'org.conscrypt.Conscrypt',
         'org.conscrypt.HandshakeListener'
 ]
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.16.Final.jar.sha1
deleted file mode 100644
index c546222971985..0000000000000
--- a/plugins/transport-nio/licenses/netty-buffer-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-63b5fa95c74785e16f2c30ce268bc222e35c8cb5
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..f8a652d0dd1b4
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-buffer-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+d6c2d13492778009d33f60e05ed90bcb535d1fd1
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.16.Final.jar.sha1
deleted file mode 100644
index 1e6c241ea0b17..0000000000000
--- a/plugins/transport-nio/licenses/netty-codec-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d84a1f21768b7309c2954521cf5a1f46c2309eb1
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..70799bf10328a
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-codec-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+a38361d893900947524f8a9da980555950e73d6a
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.16.Final.jar.sha1
deleted file mode 100644
index 71c33af1c5fc2..0000000000000
--- a/plugins/transport-nio/licenses/netty-codec-http-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d64312378b438dfdad84267c599a053327c6f02a
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..e1d34ebf89bb8
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+897100c1022c780b0a436b9349e507e8fa9800dc
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.16.Final.jar.sha1
deleted file mode 100644
index 3edf5fcea59b3..0000000000000
--- a/plugins/transport-nio/licenses/netty-common-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-177a6b30cca92f6f5f9873c9befd681377a4c328
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..bc95142683242
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-common-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+df69ce8bb9b544a71e7bbee290253cf7c93e6bad
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.16.Final.jar.sha1
deleted file mode 100644
index cba27387268d1..0000000000000
--- a/plugins/transport-nio/licenses/netty-handler-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fec0e63e7dd7f4eeef7ea8dc47a1ff32dfc7ebc2
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..80dc8b8f6fe95
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-handler-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+a035784682da0126bc25f10713dac732b5082a6d
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.16.Final.jar.sha1
deleted file mode 100644
index 3571d2ecfdc48..0000000000000
--- a/plugins/transport-nio/licenses/netty-resolver-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f6eb553b53fb3a90a8ac1170697093fed82eae28
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..afe004bd71623
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-resolver-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+f33557dcb31fa20da075ac05e4808115e32ef9b7
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.16.Final.jar.sha1
deleted file mode 100644
index e502d4c77084c..0000000000000
--- a/plugins/transport-nio/licenses/netty-transport-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3c8ee2c4d4a1cbb947a5c184c7aeb2204260958b
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.28.Final.jar.sha1
new file mode 100644
index 0000000000000..af19a16d6ed64
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-transport-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+d2ef28f49d726737f0ffe84bf66529b3bf6e0c0d
\ No newline at end of file
diff --git a/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy b/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy
index 8c8fe7c327412..1c6eb4f10e3fb 100644
--- a/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy
@@ -26,4 +26,6 @@ grant codeBase "${codebase.netty-common}" {
    // This should only currently be required as we use the netty http client for tests
    // netty makes and accepts socket connections
    permission java.net.SocketPermission "*", "accept,connect";
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };
diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/Netty4HttpClient.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/Netty4HttpClient.java
deleted file mode 100644
index becebade37348..0000000000000
--- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/Netty4HttpClient.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.http.nio;
-
-import io.netty.bootstrap.Bootstrap;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.Unpooled;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.SimpleChannelInboundHandler;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioSocketChannel;
-import io.netty.handler.codec.http.DefaultFullHttpRequest;
-import io.netty.handler.codec.http.FullHttpRequest;
-import io.netty.handler.codec.http.FullHttpResponse;
-import io.netty.handler.codec.http.HttpHeaderNames;
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpObject;
-import io.netty.handler.codec.http.HttpObjectAggregator;
-import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpRequestEncoder;
-import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.codec.http.HttpResponseDecoder;
-import io.netty.handler.codec.http.HttpVersion;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.tasks.Task;
-
-import java.io.Closeable;
-import java.net.SocketAddress;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import static io.netty.handler.codec.http.HttpHeaderNames.HOST;
-import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-
-/**
- * Tiny helper to send http requests over netty.
- */
-class Netty4HttpClient implements Closeable {
-
-    static Collection<String> returnHttpResponseBodies(Collection<FullHttpResponse> responses) {
-        List<String> list = new ArrayList<>(responses.size());
-        for (FullHttpResponse response : responses) {
-            list.add(response.content().toString(StandardCharsets.UTF_8));
-        }
-        return list;
-    }
-
-    static Collection<String> returnOpaqueIds(Collection<FullHttpResponse> responses) {
-        List<String> list = new ArrayList<>(responses.size());
-        for (HttpResponse response : responses) {
-            list.add(response.headers().get(Task.X_OPAQUE_ID));
-        }
-        return list;
-    }
-
-    private final Bootstrap clientBootstrap;
-
-    Netty4HttpClient() {
-        clientBootstrap = new Bootstrap().channel(NioSocketChannel.class).group(new NioEventLoopGroup());
-    }
-
-    public Collection<FullHttpResponse> get(SocketAddress remoteAddress, String... uris) throws InterruptedException {
-        Collection<HttpRequest> requests = new ArrayList<>(uris.length);
-        for (int i = 0; i < uris.length; i++) {
-            final HttpRequest httpRequest = new DefaultFullHttpRequest(HTTP_1_1, HttpMethod.GET, uris[i]);
-            httpRequest.headers().add(HOST, "localhost");
-            httpRequest.headers().add(Task.X_OPAQUE_ID, String.valueOf(i));
-            requests.add(httpRequest);
-        }
-        return sendRequests(remoteAddress, requests);
-    }
-
-    @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods.
-    public final Collection<FullHttpResponse> post(SocketAddress remoteAddress, Tuple<String, CharSequence>... urisAndBodies)
-        throws InterruptedException {
-        return processRequestsWithBody(HttpMethod.POST, remoteAddress, urisAndBodies);
-    }
-
-    public final FullHttpResponse post(SocketAddress remoteAddress, FullHttpRequest httpRequest) throws InterruptedException {
-        Collection<FullHttpResponse> responses = sendRequests(remoteAddress, Collections.singleton(httpRequest));
-        assert responses.size() == 1 : "expected 1 and only 1 http response";
-        return responses.iterator().next();
-    }
-
-    @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods.
-    public final Collection<FullHttpResponse> put(SocketAddress remoteAddress, Tuple<String, CharSequence>... urisAndBodies)
-        throws InterruptedException {
-        return processRequestsWithBody(HttpMethod.PUT, remoteAddress, urisAndBodies);
-    }
-
-    private Collection<FullHttpResponse> processRequestsWithBody(HttpMethod method, SocketAddress remoteAddress, Tuple<String,
-        CharSequence>... urisAndBodies) throws InterruptedException {
-        Collection<HttpRequest> requests = new ArrayList<>(urisAndBodies.length);
-        for (Tuple<String, CharSequence> uriAndBody : urisAndBodies) {
-            ByteBuf content = Unpooled.copiedBuffer(uriAndBody.v2(), StandardCharsets.UTF_8);
-            HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, method, uriAndBody.v1(), content);
-            request.headers().add(HttpHeaderNames.HOST, "localhost");
-            request.headers().add(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes());
-            request.headers().add(HttpHeaderNames.CONTENT_TYPE, "application/json");
-            requests.add(request);
-        }
-        return sendRequests(remoteAddress, requests);
-    }
-
-    private synchronized Collection<FullHttpResponse> sendRequests(
-        final SocketAddress remoteAddress,
-        final Collection<HttpRequest> requests) throws InterruptedException {
-        final CountDownLatch latch = new CountDownLatch(requests.size());
-        final Collection<FullHttpResponse> content = Collections.synchronizedList(new ArrayList<>(requests.size()));
-
-        clientBootstrap.handler(new CountDownLatchHandler(latch, content));
-
-        ChannelFuture channelFuture = null;
-        try {
-            channelFuture = clientBootstrap.connect(remoteAddress);
-            channelFuture.sync();
-
-            for (HttpRequest request : requests) {
-                channelFuture.channel().writeAndFlush(request);
-            }
-            latch.await(30, TimeUnit.SECONDS);
-
-        } finally {
-            if (channelFuture != null) {
-                channelFuture.channel().close().sync();
-            }
-        }
-
-        return content;
-    }
-
-    @Override
-    public void close() {
-        clientBootstrap.config().group().shutdownGracefully().awaitUninterruptibly();
-    }
-
-    /**
-     * helper factory which adds returned data to a list and uses a count down latch to decide when done
-     */
-    private static class CountDownLatchHandler extends ChannelInitializer<SocketChannel> {
-
-        private final CountDownLatch latch;
-        private final Collection<FullHttpResponse> content;
-
-        CountDownLatchHandler(final CountDownLatch latch, final Collection<FullHttpResponse> content) {
-            this.latch = latch;
-            this.content = content;
-        }
-
-        @Override
-        protected void initChannel(SocketChannel ch) throws Exception {
-            final int maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB).bytesAsInt();
-            ch.pipeline().addLast(new HttpResponseDecoder());
-            ch.pipeline().addLast(new HttpRequestEncoder());
-            ch.pipeline().addLast(new HttpObjectAggregator(maxContentLength));
-            ch.pipeline().addLast(new SimpleChannelInboundHandler<HttpObject>() {
-                @Override
-                protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception {
-                    final FullHttpResponse response = (FullHttpResponse) msg;
-                    content.add(response.copy());
-                    latch.countDown();
-                }
-
-                @Override
-                public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
-                    super.exceptionCaught(ctx, cause);
-                    latch.countDown();
-                }
-            });
-        }
-
-    }
-
-}
diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpClient.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpClient.java
new file mode 100644
index 0000000000000..785b6ea59701c
--- /dev/null
+++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpClient.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.http.nio;
+
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandler;
+import io.netty.handler.codec.http.DefaultFullHttpRequest;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpObjectAggregator;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpRequestEncoder;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.HttpResponseDecoder;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.nio.BytesChannelContext;
+import org.elasticsearch.nio.ChannelFactory;
+import org.elasticsearch.nio.EventHandler;
+import org.elasticsearch.nio.FlushOperation;
+import org.elasticsearch.nio.InboundChannelBuffer;
+import org.elasticsearch.nio.NioGroup;
+import org.elasticsearch.nio.NioSelector;
+import org.elasticsearch.nio.NioServerSocketChannel;
+import org.elasticsearch.nio.NioSocketChannel;
+import org.elasticsearch.nio.ReadWriteHandler;
+import org.elasticsearch.nio.SocketChannelContext;
+import org.elasticsearch.nio.WriteOperation;
+import org.elasticsearch.tasks.Task;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.ServerSocketChannel;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+import static io.netty.handler.codec.http.HttpHeaderNames.HOST;
+import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
+import static org.junit.Assert.fail;
+
+/**
+ * Tiny helper to send http requests over nio.
+ */
+class NioHttpClient implements Closeable {
+
+    static Collection<String> returnOpaqueIds(Collection<FullHttpResponse> responses) {
+        List<String> list = new ArrayList<>(responses.size());
+        for (HttpResponse response : responses) {
+            list.add(response.headers().get(Task.X_OPAQUE_ID));
+        }
+        return list;
+    }
+
+    private final NioGroup nioGroup;
+    private final Logger logger;
+
+    NioHttpClient() {
+        logger = Loggers.getLogger(NioHttpClient.class, Settings.EMPTY);
+        try {
+            nioGroup = new NioGroup(daemonThreadFactory(Settings.EMPTY, "nio-http-client"), 1,
+                (s) -> new EventHandler(this::onException, s));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public Collection<FullHttpResponse> get(InetSocketAddress remoteAddress, String... uris) throws InterruptedException {
+        Collection<HttpRequest> requests = new ArrayList<>(uris.length);
+        for (int i = 0; i < uris.length; i++) {
+            final HttpRequest httpRequest = new DefaultFullHttpRequest(HTTP_1_1, HttpMethod.GET, uris[i]);
+            httpRequest.headers().add(HOST, "localhost");
+            httpRequest.headers().add(Task.X_OPAQUE_ID, String.valueOf(i));
+            requests.add(httpRequest);
+        }
+        return sendRequests(remoteAddress, requests);
+    }
+
+    public final FullHttpResponse post(InetSocketAddress remoteAddress, FullHttpRequest httpRequest) throws InterruptedException {
+        Collection<FullHttpResponse> responses = sendRequests(remoteAddress, Collections.singleton(httpRequest));
+        assert responses.size() == 1 : "expected 1 and only 1 http response";
+        return responses.iterator().next();
+    }
+
+    private void onException(Exception e) {
+        logger.error("Exception from http client", e);
+    }
+
+    private synchronized Collection<FullHttpResponse> sendRequests(InetSocketAddress remoteAddress, Collection<HttpRequest> requests)
+        throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(requests.size());
+        final Collection<FullHttpResponse> content = Collections.synchronizedList(new ArrayList<>(requests.size()));
+
+        ChannelFactory<NioServerSocketChannel, NioSocketChannel> factory = new ClientChannelFactory(latch, content);
+
+        NioSocketChannel nioSocketChannel = null;
+        try {
+            nioSocketChannel = nioGroup.openChannel(remoteAddress, factory);
+            PlainActionFuture<Void> connectFuture = PlainActionFuture.newFuture();
+            nioSocketChannel.addConnectListener(ActionListener.toBiConsumer(connectFuture));
+            connectFuture.actionGet();
+
+            for (HttpRequest request : requests) {
+                nioSocketChannel.getContext().sendMessage(request, (v, e) -> {});
+            }
+            if (latch.await(30L, TimeUnit.SECONDS) == false) {
+                fail("Failed to get all expected responses.");
+            }
+
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        } finally {
+            if (nioSocketChannel != null) {
+                nioSocketChannel.close();
+            }
+        }
+
+        return content;
+    }
+
+    @Override
+    public void close() {
+        IOUtils.closeWhileHandlingException(nioGroup::close);
+    }
+
+    private class ClientChannelFactory extends ChannelFactory<NioServerSocketChannel, NioSocketChannel> {
+
+        private final CountDownLatch latch;
+        private final Collection<FullHttpResponse> content;
+
+        private ClientChannelFactory(CountDownLatch latch, Collection<FullHttpResponse> content) {
+            super(new RawChannelFactory(NetworkService.TCP_NO_DELAY.get(Settings.EMPTY),
+                NetworkService.TCP_KEEP_ALIVE.get(Settings.EMPTY),
+                NetworkService.TCP_REUSE_ADDRESS.get(Settings.EMPTY),
+                Math.toIntExact(NetworkService.TCP_SEND_BUFFER_SIZE.get(Settings.EMPTY).getBytes()),
+                Math.toIntExact(NetworkService.TCP_RECEIVE_BUFFER_SIZE.get(Settings.EMPTY).getBytes())));
+            this.latch = latch;
+            this.content = content;
+        }
+
+        @Override
+        public NioSocketChannel createChannel(NioSelector selector, java.nio.channels.SocketChannel channel) throws IOException {
+            NioSocketChannel nioSocketChannel = new NioSocketChannel(channel);
+            HttpClientHandler handler = new HttpClientHandler(nioSocketChannel, latch, content);
+            Consumer<Exception> exceptionHandler = (e) -> {
+                latch.countDown();
+                onException(e);
+                nioSocketChannel.close();
+            };
+            SocketChannelContext context = new BytesChannelContext(nioSocketChannel, selector, exceptionHandler, handler,
+                InboundChannelBuffer.allocatingInstance());
+            nioSocketChannel.setContext(context);
+            return nioSocketChannel;
+        }
+
+        @Override
+        public NioServerSocketChannel createServerChannel(NioSelector selector, ServerSocketChannel channel) {
+            throw new UnsupportedOperationException("Cannot create server channel");
+        }
+    }
+
+    private static class HttpClientHandler implements ReadWriteHandler {
+
+        private final NettyAdaptor adaptor;
+        private final CountDownLatch latch;
+        private final Collection<FullHttpResponse> content;
+
+        private HttpClientHandler(NioSocketChannel channel, CountDownLatch latch, Collection<FullHttpResponse> content) {
+            this.latch = latch;
+            this.content = content;
+            final int maxContentLength = Math.toIntExact(new ByteSizeValue(100, ByteSizeUnit.MB).getBytes());
+            List<ChannelHandler> handlers = new ArrayList<>(5);
+            handlers.add(new HttpResponseDecoder());
+            handlers.add(new HttpRequestEncoder());
+            handlers.add(new HttpObjectAggregator(maxContentLength));
+
+            adaptor = new NettyAdaptor(handlers.toArray(new ChannelHandler[0]));
+            adaptor.addCloseListener((v, e) -> channel.close());
+        }
+
+        @Override
+        public WriteOperation createWriteOperation(SocketChannelContext context, Object message, BiConsumer<Void, Exception> listener) {
+            assert message instanceof HttpRequest : "Expected type HttpRequest.class, found: " + message.getClass();
+            return new WriteOperation() {
+                @Override
+                public BiConsumer<Void, Exception> getListener() {
+                    return listener;
+                }
+
+                @Override
+                public SocketChannelContext getChannel() {
+                    return context;
+                }
+
+                @Override
+                public Object getObject() {
+                    return message;
+                }
+            };
+        }
+
+        @Override
+        public List<FlushOperation> writeToBytes(WriteOperation writeOperation) {
+            adaptor.write(writeOperation);
+            return pollFlushOperations();
+        }
+
+        @Override
+        public List<FlushOperation> pollFlushOperations() {
+            ArrayList<FlushOperation> copiedOperations = new ArrayList<>(adaptor.getOutboundCount());
+            FlushOperation flushOperation;
+            while ((flushOperation = adaptor.pollOutboundOperation()) != null) {
+                copiedOperations.add(flushOperation);
+            }
+            return copiedOperations;
+        }
+
+        @Override
+        public int consumeReads(InboundChannelBuffer channelBuffer) throws IOException {
+            int bytesConsumed = adaptor.read(channelBuffer.sliceAndRetainPagesTo(channelBuffer.getIndex()));
+            Object message;
+            while ((message = adaptor.pollInboundMessage()) != null) {
+                handleRequest(message);
+            }
+
+            return bytesConsumed;
+        }
+
+        @Override
+        public void close() throws IOException {
+            try {
+                adaptor.close();
+            } catch (Exception e) {
+                throw new IOException(e);
+            }
+        }
+
+        private void handleRequest(Object message) {
+            final FullHttpResponse response = (FullHttpResponse) message;
+            DefaultFullHttpResponse newResponse = new DefaultFullHttpResponse(response.protocolVersion(),
+                response.status(),
+                Unpooled.copiedBuffer(response.content()),
+                response.headers().copy(),
+                response.trailingHeaders().copy());
+            response.release();
+            content.add(newResponse);
+            latch.countDown();
+        }
+    }
+
+}
diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java
index a0cb74f7cd205..8acec830f11b9 100644
--- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java
+++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java
@@ -192,20 +192,29 @@ public void dispatchBadRequest(RestRequest request, RestChannel channel, ThreadC
             xContentRegistry(), dispatcher)) {
             transport.start();
             final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
-            try (Netty4HttpClient client = new Netty4HttpClient()) {
+            try (NioHttpClient client = new NioHttpClient()) {
                 final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/");
                 request.headers().set(HttpHeaderNames.EXPECT, expectation);
                 HttpUtil.setContentLength(request, contentLength);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(expectedStatus));
-                if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
-                    final FullHttpRequest continuationRequest =
-                        new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
-                    final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
-
-                    assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
-                    assertThat(new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done"));
+                try {
+                    assertThat(response.status(), equalTo(expectedStatus));
+                    if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
+                        final FullHttpRequest continuationRequest =
+                            new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
+                        final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
+                        try {
+                            assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
+                            assertThat(
+                                new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done")
+                            );
+                        } finally {
+                            continuationResponse.release();
+                        }
+                    }
+                } finally {
+                    response.release();
                 }
             }
         }
@@ -266,22 +275,26 @@ threadPool, xContentRegistry(), dispatcher)) {
             transport.start();
             final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
 
-            try (Netty4HttpClient client = new Netty4HttpClient()) {
+            try (NioHttpClient client = new NioHttpClient()) {
                 final String url = "/" + new String(new byte[maxInitialLineLength], Charset.forName("UTF-8"));
                 final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
-                assertThat(
-                    new String(response.content().array(), Charset.forName("UTF-8")),
-                    containsString("you sent a bad request and you should feel bad"));
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
+                    assertThat(
+                        new String(response.content().array(), Charset.forName("UTF-8")),
+                        containsString("you sent a bad request and you should feel bad"));
+                } finally {
+                    response.release();
+                }
             }
         }
 
         assertNotNull(causeReference.get());
         assertThat(causeReference.get(), instanceOf(TooLongFrameException.class));
     }
-    
+
 //    public void testReadTimeout() throws Exception {
 //        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
 //
diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioPipeliningIT.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioPipeliningIT.java
index 074aafd6eab4b..7b80885457f3f 100644
--- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioPipeliningIT.java
+++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioPipeliningIT.java
@@ -47,11 +47,11 @@ public void testThatNioHttpServerSupportsPipelining() throws Exception {
         TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses();
         TransportAddress transportAddress = randomFrom(boundAddresses);
 
-        try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
+        try (NioHttpClient nettyHttpClient = new NioHttpClient()) {
             Collection<FullHttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests);
             assertThat(responses, hasSize(5));
 
-            Collection<String> opaqueIds = Netty4HttpClient.returnOpaqueIds(responses);
+            Collection<String> opaqueIds = NioHttpClient.returnOpaqueIds(responses);
             assertOpaqueIdsInOrder(opaqueIds);
         }
     }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java
index 0dc9ea0a170ba..b034cd08d77d6 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java
@@ -87,7 +87,7 @@ public void testInvalidValue() {
         final boolean enforceLimits = randomBoolean();
         final IllegalArgumentException e = expectThrows(
                 IllegalArgumentException.class,
-                () -> BootstrapChecks.check(new BootstrapContext(Settings.EMPTY, null), enforceLimits, emptyList(), "testInvalidValue"));
+                () -> BootstrapChecks.check(new BootstrapContext(Settings.EMPTY, null), enforceLimits, emptyList()));
         final Matcher<String> matcher = containsString(
                 "[es.enforce.bootstrap.checks] must be [true] but was [" + value + "]");
         assertThat(e, hasToString(matcher));
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
index d4bc754689e68..ede61da1369f5 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
@@ -27,7 +27,9 @@
 import org.apache.logging.log4j.core.appender.ConsoleAppender;
 import org.apache.logging.log4j.core.appender.CountingNoOpAppender;
 import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.logging.log4j.spi.ExtendedLogger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.util.Constants;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.Randomness;
@@ -56,6 +58,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.lessThan;
@@ -297,8 +300,8 @@ public void testFindAppender() throws IOException, UserException {
     public void testPrefixLogger() throws IOException, IllegalAccessException, UserException {
         setupLogging("prefix");
 
-        final String prefix = randomBoolean() ? null : randomAlphaOfLength(16);
-        final Logger logger = Loggers.getLogger("prefix", prefix);
+        final String prefix = randomAlphaOfLength(16);
+        final Logger logger = new PrefixLogger((ExtendedLogger) LogManager.getLogger("prefix_test"), "prefix_test", prefix);
         logger.info("test");
         logger.info("{}", "test");
         final Exception e = new Exception("exception");
@@ -318,11 +321,8 @@ public void testPrefixLogger() throws IOException, IllegalAccessException, UserE
         final int expectedLogLines = 3;
         assertThat(events.size(), equalTo(expectedLogLines + stackTraceLength));
         for (int i = 0; i < expectedLogLines; i++) {
-            if (prefix == null) {
-                assertThat(events.get(i), startsWith("test"));
-            } else {
-                assertThat(events.get(i), startsWith("[" + prefix + "] test"));
-            }
+            assertThat("Contents of [" + path + "] are wrong",
+                    events.get(i), startsWith("[" + getTestName() + "]" + prefix + " test"));
         }
     }
 
@@ -331,8 +331,8 @@ public void testPrefixLoggerMarkersCanBeCollected() throws IOException, UserExce
 
         final int prefixes = 1 << 19; // to ensure enough markers that the GC should collect some when we force a GC below
         for (int i = 0; i < prefixes; i++) {
-            Loggers.getLogger("prefix" + i, "prefix" + i); // this has the side effect of caching a marker with this prefix
-
+            // this has the side effect of caching a marker with this prefix
+            new PrefixLogger((ExtendedLogger) LogManager.getLogger("prefix" + i), "prefix" + i, "prefix" + i);
         }
 
         System.gc(); // this will free the weakly referenced keys in the marker cache
@@ -357,6 +357,28 @@ public void testProperties() throws IOException, UserException {
         }
     }
 
+    public void testNoNodeNameWarning() throws IOException, UserException {
+        setupLogging("no_node_name");
+
+        final String path =
+            System.getProperty("es.logs.base_path") +
+                System.getProperty("file.separator") +
+                System.getProperty("es.logs.cluster_name") + ".log";
+        final List<String> events = Files.readAllLines(PathUtils.get(path));
+        assertThat(events.size(), equalTo(2));
+        final String location = "org.elasticsearch.common.logging.LogConfigurator";
+        // the first message is a warning for unsupported configuration files
+        assertLogLine(events.get(0), Level.WARN, location, "\\[null\\] Some logging configurations have %marker but don't "
+                + "have %node_name. We will automatically add %node_name to the pattern to ease the migration for users "
+                + "who customize log4j2.properties but will stop this behavior in 7.0. You should manually replace "
+                + "`%node_name` with `\\[%node_name\\]%marker ` in these locations:");
+        if (Constants.WINDOWS) {
+            assertThat(events.get(1), endsWith("no_node_name\\log4j2.properties"));
+        } else {
+            assertThat(events.get(1), endsWith("no_node_name/log4j2.properties"));
+        }
+    }
+
     private void setupLogging(final String config) throws IOException, UserException {
         setupLogging(config, Settings.EMPTY);
     }
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
index 8b956421458d0..2c9f48a359a46 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
@@ -1,13 +1,13 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 appender.file.type = File
 appender.file.name = file
 appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = [%p][%l] %marker%m%n
+appender.file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
@@ -23,10 +23,9 @@ appender.deprecation_file.type = File
 appender.deprecation_file.name = deprecation_file
 appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_file.layout.type = PatternLayout
-appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
+appender.deprecation_file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 logger.deprecation.name = deprecation
 logger.deprecation.level = warn
 logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file
 logger.deprecation.additivity = false
-
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties
index ed794cb7c3b59..8a9ec160fb1e8 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties
@@ -1,7 +1,7 @@
 appender.console3.type = Console
 appender.console3.name = console3
 appender.console3.layout.type = PatternLayout
-appender.console3.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console3.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 logger.third.name = third
 logger.third.level = debug
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties
index fd7af2ce73136..388c9f9b2fc64 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties
@@ -1,13 +1,13 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 appender.file.type = File
 appender.file.name = file
 appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = [%p][%l] %marker%m%n
+appender.file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
@@ -17,7 +17,7 @@ appender.deprecation_file.type = File
 appender.deprecation_file.name = deprecation_file
 appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_file.layout.type = PatternLayout
-appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
+appender.deprecation_file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 logger.deprecation.name = deprecation
 logger.deprecation.level = warn
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/find_appender/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/find_appender/log4j2.properties
index 8553ec5e7915f..d036e571cf0c0 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/find_appender/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/find_appender/log4j2.properties
@@ -1,7 +1,7 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 logger.has_console_appender.name = has_console_appender
 logger.has_console_appender.level = trace
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties
index 5fa85b5d156b3..ee59755a2d784 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties
@@ -1,13 +1,13 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 appender.file.type = File
 appender.file.name = file
 appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = [%p][%l] %marker%m%n
+appender.file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties
new file mode 100644
index 0000000000000..fd7af2ce73136
--- /dev/null
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties
@@ -0,0 +1,25 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+
+appender.file.type = File
+appender.file.name = file
+appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = [%p][%l] %marker%m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.file.ref = file
+
+appender.deprecation_file.type = File
+appender.deprecation_file.name = deprecation_file
+appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
+appender.deprecation_file.layout.type = PatternLayout
+appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
+
+logger.deprecation.name = deprecation
+logger.deprecation.level = warn
+logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file
+logger.deprecation.additivity = false
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties
index 5dfa369a3bd25..199fddc31c681 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties
@@ -1,13 +1,13 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 appender.file.type = File
 appender.file.name = file
 appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %marker%m%n
+appender.file.layout.pattern = [%test_thread_info]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties
index 1acb65511d607..abe4a279dc820 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties
@@ -1,13 +1,13 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 appender.file.type = File
 appender.file.name = file
 appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = [%p][%l] %marker%m%n
+appender.file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
@@ -17,7 +17,7 @@ appender.deprecation_file.type = File
 appender.deprecation_file.name = deprecation_file
 appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_file.layout.type = PatternLayout
-appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
+appender.deprecation_file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 logger.deprecation.name = org.elasticsearch.deprecation.common.settings
 logger.deprecation.level = warn
diff --git a/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats b/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats
index cb9e6658d3d99..a7628d08bbaff 100644
--- a/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats
+++ b/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats
@@ -189,7 +189,10 @@ setup() {
 
 @test "[SYSTEMD] start Elasticsearch with custom JVM options" {
     assert_file_exist $ESENVFILE
-    local temp=`mktemp -d`
+    # The custom config directory is not under /tmp or /var/tmp because
+    # systemd's private temp directory functionaly means different
+    # processes can have different views of what's in these directories
+    local temp=`mktemp -p /etc -d`
     cp "$ESCONFIG"/elasticsearch.yml "$temp"
     cp "$ESCONFIG"/log4j2.properties "$temp"
     touch "$temp/jvm.options"
diff --git a/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash
index e258c4db5e636..8fd6bd9ad3f15 100644
--- a/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash
@@ -92,11 +92,14 @@ fi
 
 @test "[$GROUP] install a sample plugin with a symlinked plugins path" {
     # Clean up after the last time this test was run
-    rm -rf /tmp/plugins.*
-    rm -rf /tmp/old_plugins.*
+    rm -rf /var/plugins.*
+    rm -rf /var/old_plugins.*
 
     rm -rf "$ESPLUGINS"
-    local es_plugins=$(mktemp -d -t 'plugins.XXXX')
+    # The custom plugins directory is not under /tmp or /var/tmp because
+    # systemd's private temp directory functionaly means different
+    # processes can have different views of what's in these directories
+    local es_plugins=$(mktemp -p /var -d -t 'plugins.XXXX')
     chown -R elasticsearch:elasticsearch "$es_plugins"
     ln -s "$es_plugins" "$ESPLUGINS"
 
diff --git a/qa/vagrant/src/test/resources/packaging/utils/utils.bash b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
index c07037a5f275b..cb71e9e6ec1a0 100644
--- a/qa/vagrant/src/test/resources/packaging/utils/utils.bash
+++ b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
@@ -555,7 +555,10 @@ run_elasticsearch_tests() {
 # Move the config directory to another directory and properly chown it.
 move_config() {
     local oldConfig="$ESCONFIG"
-    export ESCONFIG="${1:-$(mktemp -d -t 'config.XXXX')}"
+    # The custom config directory is not under /tmp or /var/tmp because
+    # systemd's private temp directory functionaly means different
+    # processes can have different views of what's in these directories
+    export ESCONFIG="${1:-$(mktemp -p /etc -d -t 'config.XXXX')}"
     echo "Moving configuration directory from $oldConfig to $ESCONFIG"
 
     # Move configuration files to the new configuration directory
diff --git a/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java b/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java
index 46fafebeb4e2d..1da713ea9bcd9 100644
--- a/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java
+++ b/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java
@@ -28,6 +28,7 @@
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestRuleLimitSysouts;
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterModule;
@@ -50,6 +51,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
+@TestRuleLimitSysouts.Limit(bytes = 14000)
 public class WildflyIT extends LuceneTestCase {
 
     public void testTransportClient() throws URISyntaxException, IOException {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml
index 29f048068b4a0..0ff5708b93efe 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml
@@ -29,3 +29,40 @@
   - is_true: nodes.fs
   - is_true: nodes.plugins
   - is_true: nodes.network_types
+
+---
+"get cluster stats returns cluster_uuid at the top level":
+  - skip:
+      version: " - 6.99.99"
+      reason:  "cluster stats including cluster_uuid at the top level is new in v6.5.0 and higher"
+
+  - do:
+      cluster.stats: {}
+
+  - is_true: cluster_uuid
+  - is_true: timestamp
+  - is_true: cluster_name
+  - match: {status: green}
+  - gte: { indices.count: 0}
+  - is_true: indices.docs
+  - is_true: indices.store
+  - is_true: indices.fielddata
+  - is_true: indices.query_cache
+  - is_true: indices.completion
+  - is_true: indices.segments
+  - gte: { nodes.count.total: 1}
+  - gte: { nodes.count.master: 1}
+  - gte: { nodes.count.data: 1}
+  - gte: { nodes.count.ingest: 0}
+  - gte: { nodes.count.coordinating_only: 0}
+  - is_true: nodes.os
+  - is_true: nodes.os.mem.total_in_bytes
+  - is_true: nodes.os.mem.free_in_bytes
+  - is_true: nodes.os.mem.used_in_bytes
+  - gte: { nodes.os.mem.free_percent: 0 }
+  - gte: { nodes.os.mem.used_percent: 0 }
+  - is_true: nodes.process
+  - is_true: nodes.jvm
+  - is_true: nodes.fs
+  - is_true: nodes.plugins
+  - is_true: nodes.network_types
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
index 2dfd868d66b06..521dc4c1cac8d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
@@ -1,7 +1,13 @@
 setup:
   - do:
       indices.create:
-          index:  test
+        index:  test
+        body:
+          mappings:
+            test:
+              properties:
+                numeric_group: { type: integer }
+
   - do:
       index:
           index:  test
@@ -341,3 +347,33 @@ setup:
   - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._version: 55 }
   - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
   - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._version: 44 }
+
+---
+"field collapsing on a field alias":
+  - skip:
+      version: " - 6.3.99"
+      reason: Field aliases were introduced in 6.4.0.
+  - do:
+      indices.put_mapping:
+        index: test
+        type: test
+        body:
+          test:
+            properties:
+              group_alias: { type: alias, path: numeric_group }
+  - do:
+      search:
+        index: test
+        body:
+          collapse: { field: group_alias, inner_hits: { name: sub_hits } }
+          sort: [{ sort: desc }]
+
+  - match: { hits.total: 6 }
+  - length: { hits.hits: 3 }
+
+  - match: { hits.hits.0.fields.group_alias: [3] }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.total: 1}
+  - match: { hits.hits.1.fields.group_alias: [1] }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.total: 3}
+  - match: { hits.hits.2.fields.group_alias: [25] }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.total: 2}
diff --git a/server/build.gradle b/server/build.gradle
index deb3839897918..1964eddd03e95 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -220,7 +220,6 @@ thirdPartyAudit.excludes = [
   'com.fasterxml.jackson.dataformat.xml.JacksonXmlModule',
   'com.fasterxml.jackson.dataformat.xml.XmlMapper',
   'com.fasterxml.jackson.dataformat.xml.util.DefaultXmlPrettyPrinter',
-  'com.fasterxml.jackson.databind.node.JsonNodeFactory',
   'com.fasterxml.jackson.databind.node.ObjectNode',
   'org.fusesource.jansi.Ansi',
   'org.fusesource.jansi.AnsiRenderer$Code',
@@ -262,12 +261,6 @@ thirdPartyAudit.excludes = [
   'javax.mail.internet.MimeMultipart',
   'javax.mail.internet.MimeUtility',
   'javax.mail.util.ByteArrayDataSource',
-  'javax.persistence.AttributeConverter',
-  'javax.persistence.EntityManager',
-  'javax.persistence.EntityManagerFactory',
-  'javax.persistence.EntityTransaction',
-  'javax.persistence.Persistence',
-  'javax.persistence.PersistenceException',
   'org.apache.commons.compress.compressors.CompressorStreamFactory',
   'org.apache.commons.compress.utils.IOUtils',
   'org.apache.commons.csv.CSVFormat',
@@ -311,6 +304,16 @@ thirdPartyAudit.excludes = [
   'com.google.common.geometry.S2LatLng',
 ]
 
+if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+  // Used by Log4J 2.11.1
+  thirdPartyAudit.excludes += [
+    'java.io.ObjectInputFilter',
+    'java.io.ObjectInputFilter$Config',
+    'java.io.ObjectInputFilter$FilterInfo',
+    'java.io.ObjectInputFilter$Status'
+  ]
+}
+
 if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
   thirdPartyAudit.excludes += ['javax.xml.bind.DatatypeConverter']
 }
diff --git a/server/licenses/log4j-1.2-api-2.11.1.jar.sha1 b/server/licenses/log4j-1.2-api-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..575d75dbda8c5
--- /dev/null
+++ b/server/licenses/log4j-1.2-api-2.11.1.jar.sha1
@@ -0,0 +1 @@
+3aba3398fe064a3eab4331f88161c7480e848418
\ No newline at end of file
diff --git a/server/licenses/log4j-1.2-api-2.9.1.jar.sha1 b/server/licenses/log4j-1.2-api-2.9.1.jar.sha1
deleted file mode 100644
index 0b5acc62b7a13..0000000000000
--- a/server/licenses/log4j-1.2-api-2.9.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-894f96d677880d4ab834a1356f62b875e579caaa
\ No newline at end of file
diff --git a/server/licenses/log4j-api-2.11.1.jar.sha1 b/server/licenses/log4j-api-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..4b1bfffac179f
--- /dev/null
+++ b/server/licenses/log4j-api-2.11.1.jar.sha1
@@ -0,0 +1 @@
+268f0fe4df3eefe052b57c87ec48517d64fb2a10
\ No newline at end of file
diff --git a/server/licenses/log4j-api-2.9.1.jar.sha1 b/server/licenses/log4j-api-2.9.1.jar.sha1
deleted file mode 100644
index e1a89fadfed95..0000000000000
--- a/server/licenses/log4j-api-2.9.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7a2999229464e7a324aa503c0a52ec0f05efe7bd
\ No newline at end of file
diff --git a/server/licenses/log4j-core-2.11.1.jar.sha1 b/server/licenses/log4j-core-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..2fb8589380a03
--- /dev/null
+++ b/server/licenses/log4j-core-2.11.1.jar.sha1
@@ -0,0 +1 @@
+592a48674c926b01a9a747c7831bcd82a9e6d6e4
\ No newline at end of file
diff --git a/server/licenses/log4j-core-2.9.1.jar.sha1 b/server/licenses/log4j-core-2.9.1.jar.sha1
deleted file mode 100644
index 990ea322a7613..0000000000000
--- a/server/licenses/log4j-core-2.9.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c041978c686866ee8534f538c6220238db3bb6be
\ No newline at end of file
diff --git a/server/licenses/lucene-analyzers-common-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-analyzers-common-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..2cbf39687624c
--- /dev/null
+++ b/server/licenses/lucene-analyzers-common-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+c547b30525ad80d0ceeaa40c2d3a901c7e76fd46
\ No newline at end of file
diff --git a/server/licenses/lucene-analyzers-common-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-analyzers-common-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index 8b2a098a3a2eb..0000000000000
--- a/server/licenses/lucene-analyzers-common-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d27958843ca118db2ffd2c242ae3761bd5a47328
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-backward-codecs-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..9e2473361f033
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+9c327295d54d5abd2684e00c3aefe58aa1caace7
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-backward-codecs-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index d8496a0a86ae2..0000000000000
--- a/server/licenses/lucene-backward-codecs-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7ea220ba8e4accb8b04e280463042ad470e23bc0
\ No newline at end of file
diff --git a/server/licenses/lucene-core-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-core-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..fdedaf3fc5756
--- /dev/null
+++ b/server/licenses/lucene-core-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+73dd7703a94ec2357581f65ee7c1c4d618ff310f
\ No newline at end of file
diff --git a/server/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index d38fb392c350b..0000000000000
--- a/server/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-471096d6e92338b208aa91f3a85feb2f9cfc4afd
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-grouping-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..4e555692b0f9a
--- /dev/null
+++ b/server/licenses/lucene-grouping-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+1c3802fa30990a1758f2df19d17fe2c95fc45870
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-grouping-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index 7f83082fa0c1d..0000000000000
--- a/server/licenses/lucene-grouping-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f0af947c60d24f779c22f774e81ebd7dd91cc932
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-highlighter-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..73b6c15f332f9
--- /dev/null
+++ b/server/licenses/lucene-highlighter-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+8d7abdbb7900d7e6a76c391d8be07217c0d882ca
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-highlighter-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index 6b9f2cb724dd0..0000000000000
--- a/server/licenses/lucene-highlighter-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fbc83ac5a0139ed7e7faf6c95a2718f46f28c641
\ No newline at end of file
diff --git a/server/licenses/lucene-join-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-join-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..23414b8e8e134
--- /dev/null
+++ b/server/licenses/lucene-join-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+011f78ae9d9a386fcf20ceea29ba30e75fb512e8
\ No newline at end of file
diff --git a/server/licenses/lucene-join-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-join-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index a085943140ec2..0000000000000
--- a/server/licenses/lucene-join-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-30adfe493982b0db059dc243e269eea38d850d46
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-memory-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..d227ebaf46368
--- /dev/null
+++ b/server/licenses/lucene-memory-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+c3dd461a7cebdcacc77304660218513e10f89adb
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-memory-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index 7acc70be15182..0000000000000
--- a/server/licenses/lucene-memory-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-656f304261d9aad05070fb68593beffafe9147e3
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-misc-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..a892f3a2272ba
--- /dev/null
+++ b/server/licenses/lucene-misc-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+d63101181708d78eccc441b0d1193dd91d1a0bf1
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-misc-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index e46b138ba7b21..0000000000000
--- a/server/licenses/lucene-misc-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8bf22ad81a7480c255b55bada401eb131bfdb4df
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-queries-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..5d0fead48cbc9
--- /dev/null
+++ b/server/licenses/lucene-queries-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+22e56fbd44d6a47d7dddbdda3c17ce22ad0a6680
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-queries-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index a7114feef6282..0000000000000
--- a/server/licenses/lucene-queries-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-edb3de4d68a34c1e1ca08f79fe4d103b10e98ad1
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-queryparser-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..8be3d6447b0bb
--- /dev/null
+++ b/server/licenses/lucene-queryparser-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+36b38a1d71045f5bee5dc40526f8d57084dbdc00
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-queryparser-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index cf3011c9a45d0..0000000000000
--- a/server/licenses/lucene-queryparser-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7ece30d5f1e18d96f61644451c858c3d9960558f
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-sandbox-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..6d968f5400c52
--- /dev/null
+++ b/server/licenses/lucene-sandbox-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+21eb8b111bcb94f4abb8c6402dfd10f51ecc0b38
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-sandbox-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index 30513e58bf6bb..0000000000000
--- a/server/licenses/lucene-sandbox-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ad3bd0c2ed96556193c7215bef328e689d0b157f
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-spatial-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..b6aec2eae1dda
--- /dev/null
+++ b/server/licenses/lucene-spatial-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+d60081c5641ed21aea82d5d0976b40e1f184c8e5
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-spatial-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index 6146b055c13f0..0000000000000
--- a/server/licenses/lucene-spatial-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8a6bd97e39ee5af60126adbe8c8375dc41b1ea8e
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-spatial-extras-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..6999baccc89e9
--- /dev/null
+++ b/server/licenses/lucene-spatial-extras-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+2d42b373546aa8923d25e4e9a673dd186064f9bd
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-spatial-extras-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index c812f0449271d..0000000000000
--- a/server/licenses/lucene-spatial-extras-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-07e748d2d80000a7a213f3405b82b6e26b452948
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-spatial3d-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..b866b1985568b
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+7f31607959e5a2ed84ab2d9a007a3f76e9a2d38c
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-spatial3d-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index b5ad83ac9fe13..0000000000000
--- a/server/licenses/lucene-spatial3d-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fd737bd5562f3943618ee7e73a0aaffb6319fdb2
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/server/licenses/lucene-suggest-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..55e1c5990de63
--- /dev/null
+++ b/server/licenses/lucene-suggest-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+f7619348f0619867c52f4801531c70358f49873a
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-suggest-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index 452b96420f8d7..0000000000000
--- a/server/licenses/lucene-suggest-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ff3f260d1dc8c18bc67f3c33aa84a0ad290daac5
\ No newline at end of file
diff --git a/server/src/main/java/org/elasticsearch/ExceptionsHelper.java b/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
index dff14bc8b393b..59eb8b60dadba 100644
--- a/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
+++ b/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
@@ -39,6 +39,7 @@
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Queue;
 import java.util.Set;
@@ -278,7 +279,7 @@ public static ShardOperationFailedException[] groupBy(ShardOperationFailedExcept
         List<ShardOperationFailedException> uniqueFailures = new ArrayList<>();
         Set<GroupBy> reasons = new HashSet<>();
         for (ShardOperationFailedException failure : failures) {
-            GroupBy reason = new GroupBy(failure.getCause());
+            GroupBy reason = new GroupBy(failure);
             if (reasons.contains(reason) == false) {
                 reasons.add(reason);
                 uniqueFailures.add(failure);
@@ -287,46 +288,47 @@ public static ShardOperationFailedException[] groupBy(ShardOperationFailedExcept
         return uniqueFailures.toArray(new ShardOperationFailedException[0]);
     }
 
-    static class GroupBy {
+    private static class GroupBy {
         final String reason;
         final String index;
         final Class<? extends Throwable> causeType;
 
-        GroupBy(Throwable t) {
-            if (t instanceof ElasticsearchException) {
-                final Index index = ((ElasticsearchException) t).getIndex();
-                if (index != null) {
-                    this.index = index.getName();
-                } else {
-                    this.index = null;
+        GroupBy(ShardOperationFailedException failure) {
+            Throwable cause = failure.getCause();
+            //the index name from the failure contains the cluster alias when using CCS. Ideally failures should be grouped by
+            //index name and cluster alias. That's why the failure index name has the precedence over the one coming from the cause,
+            //which does not include the cluster alias.
+            String indexName = failure.index();
+            if (indexName == null) {
+                if (cause instanceof ElasticsearchException) {
+                    final Index index = ((ElasticsearchException) cause).getIndex();
+                    if (index != null) {
+                        indexName = index.getName();
+                    }
                 }
-            } else {
-                index = null;
             }
-            reason = t.getMessage();
-            causeType = t.getClass();
+            this.index = indexName;
+            this.reason = cause.getMessage();
+            this.causeType = cause.getClass();
         }
 
         @Override
         public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
             GroupBy groupBy = (GroupBy) o;
-
-            if (!causeType.equals(groupBy.causeType)) return false;
-            if (index != null ? !index.equals(groupBy.index) : groupBy.index != null) return false;
-            if (reason != null ? !reason.equals(groupBy.reason) : groupBy.reason != null) return false;
-
-            return true;
+            return Objects.equals(reason, groupBy.reason) &&
+                Objects.equals(index, groupBy.index) &&
+                Objects.equals(causeType, groupBy.causeType);
         }
 
         @Override
         public int hashCode() {
-            int result = reason != null ? reason.hashCode() : 0;
-            result = 31 * result + (index != null ? index.hashCode() : 0);
-            result = 31 * result + causeType.hashCode();
-            return result;
+            return Objects.hash(reason, index, causeType);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java b/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java
index 013bf06d2f8b2..08a97d4d993a6 100644
--- a/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java
+++ b/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java
@@ -19,39 +19,70 @@
 
 package org.elasticsearch.action;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.rest.RestStatus;
 
+import java.util.Objects;
+
 /**
  * An exception indicating that a failure occurred performing an operation on the shard.
  *
- *
  */
-public interface ShardOperationFailedException extends Streamable, ToXContent {
+public abstract class ShardOperationFailedException implements Streamable, ToXContent {
+
+    protected String index;
+    protected int shardId;
+    protected String reason;
+    protected RestStatus status;
+    protected Throwable cause;
+
+    protected ShardOperationFailedException() {
+
+    }
+
+    protected ShardOperationFailedException(@Nullable String index, int shardId, String reason, RestStatus status, Throwable cause) {
+        this.index = index;
+        this.shardId = shardId;
+        this.reason = Objects.requireNonNull(reason, "reason cannot be null");
+        this.status = Objects.requireNonNull(status, "status cannot be null");
+        this.cause = Objects.requireNonNull(cause, "cause cannot be null");
+    }
 
     /**
      * The index the operation failed on. Might return {@code null} if it can't be derived.
      */
-    String index();
+    @Nullable
+    public final String index() {
+        return index;
+    }
 
     /**
      * The index the operation failed on. Might return {@code -1} if it can't be derived.
      */
-    int shardId();
+    public final int shardId() {
+        return shardId;
+    }
 
     /**
      * The reason of the failure.
      */
-    String reason();
+    public final String reason() {
+        return reason;
+    }
 
     /**
      * The status of the failure.
      */
-    RestStatus status();
+    public final RestStatus status() {
+        return status;
+    }
 
     /**
      * The cause of this failure
      */
-    Throwable getCause();
+    public final Throwable getCause() {
+        return cause;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
index 095184a5a73a2..cdf122545c4e2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Locale;
 
 import static org.elasticsearch.cluster.routing.allocation.AbstractAllocationDecision.discoveryNodeToXContent;
@@ -189,7 +190,8 @@ private XContentBuilder unassignedInfoToXContent(UnassignedInfo unassignedInfo,
 
         builder.startObject("unassigned_info");
         builder.field("reason", unassignedInfo.getReason());
-        builder.field("at", UnassignedInfo.DATE_TIME_FORMATTER.printer().print(unassignedInfo.getUnassignedTimeInMillis()));
+        builder.field("at",
+            UnassignedInfo.DATE_TIME_FORMATTER.format(Instant.ofEpochMilli(unassignedInfo.getUnassignedTimeInMillis())));
         if (unassignedInfo.getNumFailedAllocations() >  0) {
             builder.field("failed_allocation_attempts", unassignedInfo.getNumFailedAllocations());
         }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
index 469106c9a6102..372704afaf666 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
@@ -40,15 +40,18 @@ public class ClusterStatsResponse extends BaseNodesResponse<ClusterStatsNodeResp
     ClusterStatsIndices indicesStats;
     ClusterHealthStatus status;
     long timestamp;
+    String clusterUUID;
 
     ClusterStatsResponse() {
     }
 
     public ClusterStatsResponse(long timestamp,
+                                String clusterUUID,
                                 ClusterName clusterName,
                                 List<ClusterStatsNodeResponse> nodes,
                                 List<FailedNodeException> failures) {
         super(clusterName, nodes, failures);
+        this.clusterUUID = clusterUUID;
         this.timestamp = timestamp;
         nodesStats = new ClusterStatsNodes(nodes);
         indicesStats = new ClusterStatsIndices(nodes);
@@ -61,6 +64,10 @@ public ClusterStatsResponse(long timestamp,
         }
     }
 
+    public String getClusterUUID() {
+        return this.clusterUUID;
+    }
+
     public long getTimestamp() {
         return this.timestamp;
     }
@@ -111,6 +118,7 @@ protected void writeNodesTo(StreamOutput out, List<ClusterStatsNodeResponse> nod
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("cluster_uuid", getClusterUUID());
         builder.field("timestamp", getTimestamp());
         if (status != null) {
             builder.field("status", status.name().toLowerCase(Locale.ROOT));
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 227b1359d4f09..66b258670c128 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -74,6 +74,7 @@ protected ClusterStatsResponse newResponse(ClusterStatsRequest request,
                                                List<ClusterStatsNodeResponse> responses, List<FailedNodeException> failures) {
         return new ClusterStatsResponse(
             System.currentTimeMillis(),
+            clusterService.state().metaData().clusterUUID(),
             clusterService.getClusterName(),
             responses,
             failures);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 3fa046263afc7..ab05d0690031b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -109,8 +109,9 @@ protected void masterOperation(final RolloverRequest rolloverRequest, final Clus
                                    final ActionListener<RolloverResponse> listener) {
         final MetaData metaData = state.metaData();
         validate(metaData, rolloverRequest);
-        final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(rolloverRequest.getAlias());
-        final IndexMetaData indexMetaData = aliasOrIndex.getIndices().get(0);
+        final AliasOrIndex.Alias alias = (AliasOrIndex.Alias) metaData.getAliasAndIndexLookup().get(rolloverRequest.getAlias());
+        final IndexMetaData indexMetaData = alias.getWriteIndex();
+        final boolean explicitWriteIndex = Boolean.TRUE.equals(indexMetaData.getAliases().get(alias.getAliasName()).writeIndex());
         final String sourceProvidedName = indexMetaData.getSettings().get(IndexMetaData.SETTING_INDEX_PROVIDED_NAME,
             indexMetaData.getIndex().getName());
         final String sourceIndexName = indexMetaData.getIndex().getName();
@@ -138,10 +139,15 @@ public void onResponse(IndicesStatsResponse statsResponse) {
                         CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(unresolvedName, rolloverIndexName,
                             rolloverRequest);
                         createIndexService.createIndex(updateRequest, ActionListener.wrap(createIndexClusterStateUpdateResponse -> {
-                            // switch the alias to point to the newly created index
-                            indexAliasesService.indicesAliases(
-                                prepareRolloverAliasesUpdateRequest(sourceIndexName, rolloverIndexName,
-                                    rolloverRequest),
+                            final IndicesAliasesClusterStateUpdateRequest aliasesUpdateRequest;
+                            if (explicitWriteIndex) {
+                                aliasesUpdateRequest = prepareRolloverAliasesWriteIndexUpdateRequest(sourceIndexName,
+                                    rolloverIndexName, rolloverRequest);
+                            } else {
+                                aliasesUpdateRequest = prepareRolloverAliasesUpdateRequest(sourceIndexName,
+                                    rolloverIndexName, rolloverRequest);
+                            }
+                            indexAliasesService.indicesAliases(aliasesUpdateRequest,
                                 ActionListener.wrap(aliasClusterStateUpdateResponse -> {
                                     if (aliasClusterStateUpdateResponse.isAcknowledged()) {
                                         clusterService.submitStateUpdateTask("update_rollover_info", new ClusterStateUpdateTask() {
@@ -196,8 +202,19 @@ public void onFailure(Exception e) {
     static IndicesAliasesClusterStateUpdateRequest prepareRolloverAliasesUpdateRequest(String oldIndex, String newIndex,
                                                                                        RolloverRequest request) {
         List<AliasAction> actions = unmodifiableList(Arrays.asList(
-                new AliasAction.Add(newIndex, request.getAlias(), null, null, null, null),
-                new AliasAction.Remove(oldIndex, request.getAlias())));
+            new AliasAction.Add(newIndex, request.getAlias(), null, null, null, null),
+            new AliasAction.Remove(oldIndex, request.getAlias())));
+        final IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest(actions)
+            .ackTimeout(request.ackTimeout())
+            .masterNodeTimeout(request.masterNodeTimeout());
+        return updateRequest;
+    }
+
+    static IndicesAliasesClusterStateUpdateRequest prepareRolloverAliasesWriteIndexUpdateRequest(String oldIndex, String newIndex,
+                                                                                                 RolloverRequest request) {
+        List<AliasAction> actions = unmodifiableList(Arrays.asList(
+            new AliasAction.Add(newIndex, request.getAlias(), null, null, null, true),
+            new AliasAction.Add(oldIndex, request.getAlias(), null, null, null, false)));
         final IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest(actions)
             .ackTimeout(request.ackTimeout())
             .masterNodeTimeout(request.masterNodeTimeout());
@@ -244,8 +261,9 @@ static void validate(MetaData metaData, RolloverRequest request) {
         if (aliasOrIndex.isAlias() == false) {
             throw new IllegalArgumentException("source alias is a concrete index");
         }
-        if (aliasOrIndex.getIndices().size() != 1) {
-            throw new IllegalArgumentException("source alias maps to multiple indices");
+        final AliasOrIndex.Alias alias = (AliasOrIndex.Alias) aliasOrIndex;
+        if (alias.getWriteIndex() == null) {
+            throw new IllegalArgumentException("source alias [" + alias.getAliasName() + "] does not point to a write index");
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
index 6cf160897482c..72aeb7f757528 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
@@ -21,7 +21,6 @@
 
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
@@ -248,7 +247,7 @@ public String nodeId() {
             return nodeId;
         }
 
-        public static Failure readFailure(StreamInput in) throws IOException {
+        static Failure readFailure(StreamInput in) throws IOException {
             Failure failure = new Failure();
             failure.readFrom(in);
             return failure;
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResultHolder.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResultHolder.java
deleted file mode 100644
index 3e7ee41b914b7..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResultHolder.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.bulk;
-
-import org.elasticsearch.action.DocWriteResponse;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.engine.VersionConflictEngineException;
-
-/**
- * A struct-like holder for a bulk items reponse, result, and the resulting
- * replica operation to be executed.
- */
-class BulkItemResultHolder {
-    public final @Nullable DocWriteResponse response;
-    public final @Nullable Engine.Result operationResult;
-    public final BulkItemRequest replicaRequest;
-
-    BulkItemResultHolder(@Nullable DocWriteResponse response,
-                         @Nullable Engine.Result operationResult,
-                         BulkItemRequest replicaRequest) {
-        this.response = response;
-        this.operationResult = operationResult;
-        this.replicaRequest = replicaRequest;
-    }
-
-    public boolean isVersionConflict() {
-        return operationResult == null ? false :
-                operationResult.getFailure() instanceof VersionConflictEngineException;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java
new file mode 100644
index 0000000000000..5f61d90d500e7
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.action.support.replication.TransportWriteAction;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.translog.Translog;
+
+import java.util.Arrays;
+
+/**
+ * This is a utility class that holds the per request state needed to perform bulk operations on the primary.
+ * More specifically, it maintains an index to the current executing bulk item, which allows execution
+ * to stop and wait for external events such as mapping updates.
+ */
+class BulkPrimaryExecutionContext {
+
+    enum ItemProcessingState {
+        /** Item execution is ready to start, no operations have been performed yet */
+        INITIAL,
+        /**
+         * The incoming request has been translated to a request that can be executed on the shard.
+         * This is used to convert update requests to a fully specified index or delete requests.
+         */
+        TRANSLATED,
+        /**
+         * the request can not execute with the current mapping and should wait for a new mapping
+         * to arrive from the master. A mapping request for the needed changes has already been
+         * submitted
+         */
+        WAIT_FOR_MAPPING_UPDATE,
+        /**
+         * The request should be executed again, but there is no need to wait for an external event.
+         * This is needed to support retry on conflicts during updates.
+         */
+        IMMEDIATE_RETRY,
+        /** The request has been executed on the primary shard (successfully or not) */
+        EXECUTED,
+        /**
+         * No further handling of current request is needed. The result has been converted to a user response
+         * and execution can continue to the next item (if available).
+         */
+        COMPLETED
+    }
+
+    private final BulkShardRequest request;
+    private final IndexShard primary;
+    private Translog.Location locationToSync = null;
+    private int currentIndex = -1;
+
+    private ItemProcessingState currentItemState;
+    private DocWriteRequest requestToExecute;
+    private BulkItemResponse executionResult;
+    private int retryCounter;
+
+
+    BulkPrimaryExecutionContext(BulkShardRequest request, IndexShard primary) {
+        this.request = request;
+        this.primary = primary;
+        advance();
+    }
+
+
+    private int findNextNonAborted(int startIndex) {
+        final int length = request.items().length;
+        while (startIndex < length && isAborted(request.items()[startIndex].getPrimaryResponse())) {
+            startIndex++;
+        }
+        return startIndex;
+    }
+
+    private static boolean isAborted(BulkItemResponse response) {
+        return response != null && response.isFailed() && response.getFailure().isAborted();
+    }
+
+    /** move to the next item to execute */
+    private void advance() {
+        assert currentItemState == ItemProcessingState.COMPLETED || currentIndex == -1 :
+            "moving to next but current item wasn't completed (state: " + currentItemState + ")";
+        currentItemState = ItemProcessingState.INITIAL;
+        currentIndex =  findNextNonAborted(currentIndex + 1);
+        retryCounter = 0;
+        requestToExecute = null;
+        executionResult = null;
+        assert assertInvariants(ItemProcessingState.INITIAL);
+    }
+
+    /** gets the current, untranslated item request */
+    public DocWriteRequest<?> getCurrent() {
+        return getCurrentItem().request();
+    }
+
+    public BulkShardRequest getBulkShardRequest() {
+        return request;
+    }
+
+    /** returns the result of the request that has been executed on the shard */
+    public BulkItemResponse getExecutionResult() {
+        assert assertInvariants(ItemProcessingState.EXECUTED);
+        return executionResult;
+    }
+
+    /** returns the number of times the current operation has been retried */
+    public int getRetryCounter() {
+        return retryCounter;
+    }
+
+    /** returns true if the current request has been executed on the primary */
+    public boolean isOperationExecuted() {
+        return currentItemState == ItemProcessingState.EXECUTED;
+    }
+
+    /** returns true if the request needs to wait for a mapping update to arrive from the master */
+    public boolean requiresWaitingForMappingUpdate() {
+        return currentItemState == ItemProcessingState.WAIT_FOR_MAPPING_UPDATE;
+    }
+
+    /** returns true if the current request should be retried without waiting for an external event */
+    public boolean requiresImmediateRetry() {
+        return currentItemState == ItemProcessingState.IMMEDIATE_RETRY;
+    }
+
+    /**
+     * returns true if the current request has been completed and it's result translated to a user
+     * facing response
+     */
+    public boolean isCompleted() {
+        return currentItemState == ItemProcessingState.COMPLETED;
+    }
+
+    /**
+     * returns true if the current request is in INITIAL state
+     */
+    public boolean isInitial() {
+        return currentItemState == ItemProcessingState.INITIAL;
+    }
+
+    /**
+     * returns true if {@link #advance()} has moved the current item beyond the
+     * end of the {@link BulkShardRequest#items()} array.
+     */
+    public boolean hasMoreOperationsToExecute() {
+        return currentIndex < request.items().length;
+    }
+
+
+    /** returns the name of the index the current request used */
+    public String getConcreteIndex() {
+        return getCurrentItem().index();
+    }
+
+    /** returns any primary response that was set by a previous primary */
+    public BulkItemResponse getPreviousPrimaryResponse() {
+        return getCurrentItem().getPrimaryResponse();
+    }
+
+    /** returns a translog location that is needed to be synced in order to persist all operations executed so far */
+    public Translog.Location getLocationToSync() {
+        assert hasMoreOperationsToExecute() == false;
+        // we always get to the end of the list by using advance, which in turn sets the state to INITIAL
+        assert assertInvariants(ItemProcessingState.INITIAL);
+        return locationToSync;
+    }
+
+    private BulkItemRequest getCurrentItem() {
+        return request.items()[currentIndex];
+    }
+
+    /** returns the primary shard */
+    public IndexShard getPrimary() {
+        return primary;
+    }
+
+    /**
+     * sets the request that should actually be executed on the primary. This can be different then the request
+     * received from the user (specifically, an update request is translated to an indexing or delete request).
+     */
+    public void setRequestToExecute(DocWriteRequest writeRequest) {
+        assert assertInvariants(ItemProcessingState.INITIAL);
+        requestToExecute = writeRequest;
+        currentItemState = ItemProcessingState.TRANSLATED;
+        assert assertInvariants(ItemProcessingState.TRANSLATED);
+    }
+
+    /** returns the request that should be executed on the shard. */
+    public <T extends DocWriteRequest<T>> T getRequestToExecute() {
+        assert assertInvariants(ItemProcessingState.TRANSLATED);
+        return (T) requestToExecute;
+    }
+
+    /** indicates that the current operation can not be completed and needs to wait for a new mapping from the master */
+    public void markAsRequiringMappingUpdate() {
+        assert assertInvariants(ItemProcessingState.TRANSLATED);
+        currentItemState = ItemProcessingState.WAIT_FOR_MAPPING_UPDATE;
+        requestToExecute = null;
+        assert assertInvariants(ItemProcessingState.WAIT_FOR_MAPPING_UPDATE);
+    }
+
+    /** resets the current item state, prepare for a new execution */
+    public void resetForExecutionForRetry() {
+        assertInvariants(ItemProcessingState.WAIT_FOR_MAPPING_UPDATE, ItemProcessingState.EXECUTED);
+        currentItemState = ItemProcessingState.INITIAL;
+        requestToExecute = null;
+        executionResult = null;
+        assertInvariants(ItemProcessingState.INITIAL);
+    }
+
+    /** completes the operation without doing anything on the primary */
+    public void markOperationAsNoOp(DocWriteResponse response) {
+        assertInvariants(ItemProcessingState.INITIAL);
+        executionResult = new BulkItemResponse(getCurrentItem().id(), getCurrentItem().request().opType(), response);
+        currentItemState = ItemProcessingState.EXECUTED;
+        assertInvariants(ItemProcessingState.EXECUTED);
+    }
+
+    /** indicates that the operation needs to be failed as the required mapping didn't arrive in time */
+    public void failOnMappingUpdate(Exception cause) {
+        assert assertInvariants(ItemProcessingState.WAIT_FOR_MAPPING_UPDATE);
+        currentItemState = ItemProcessingState.EXECUTED;
+        final DocWriteRequest docWriteRequest = getCurrentItem().request();
+        executionResult = new BulkItemResponse(getCurrentItem().id(), docWriteRequest.opType(),
+            // Make sure to use getCurrentItem().index() here, if you use docWriteRequest.index() it will use the
+            // concrete index instead of an alias if used!
+            new BulkItemResponse.Failure(getCurrentItem().index(), docWriteRequest.type(), docWriteRequest.id(), cause));
+        markAsCompleted(executionResult);
+    }
+
+    /** the current operation has been executed on the primary with the specified result */
+    public void markOperationAsExecuted(Engine.Result result) {
+        assertInvariants(ItemProcessingState.TRANSLATED);
+        final BulkItemRequest current = getCurrentItem();
+        DocWriteRequest docWriteRequest = getRequestToExecute();
+        switch (result.getResultType()) {
+            case SUCCESS:
+                final DocWriteResponse response;
+                if (result.getOperationType() == Engine.Operation.TYPE.INDEX) {
+                    Engine.IndexResult indexResult = (Engine.IndexResult) result;
+                    response = new IndexResponse(primary.shardId(), requestToExecute.type(), requestToExecute.id(),
+                        result.getSeqNo(), result.getTerm(), indexResult.getVersion(), indexResult.isCreated());
+                } else if (result.getOperationType() == Engine.Operation.TYPE.DELETE) {
+                    Engine.DeleteResult deleteResult = (Engine.DeleteResult) result;
+                    response = new DeleteResponse(primary.shardId(), requestToExecute.type(), requestToExecute.id(),
+                        deleteResult.getSeqNo(), result.getTerm(), deleteResult.getVersion(), deleteResult.isFound());
+
+                } else {
+                    throw new AssertionError("unknown result type :" + result.getResultType());
+                }
+                executionResult = new BulkItemResponse(current.id(), current.request().opType(), response);
+                // set a blank ShardInfo so we can safely send it to the replicas. We won't use it in the real response though.
+                executionResult.getResponse().setShardInfo(new ReplicationResponse.ShardInfo());
+                locationToSync = TransportWriteAction.locationToSync(locationToSync, result.getTranslogLocation());
+                break;
+            case FAILURE:
+                executionResult = new BulkItemResponse(current.id(), docWriteRequest.opType(),
+                    // Make sure to use request.index() here, if you
+                    // use docWriteRequest.index() it will use the
+                    // concrete index instead of an alias if used!
+                    new BulkItemResponse.Failure(request.index(), docWriteRequest.type(), docWriteRequest.id(),
+                        result.getFailure(), result.getSeqNo()));
+                break;
+            default:
+                throw new AssertionError("unknown result type for " + getCurrentItem() + ": " + result.getResultType());
+        }
+        currentItemState = ItemProcessingState.EXECUTED;
+    }
+
+    /** finishes the execution of the current request, with the response that should be returned to the user */
+    public void markAsCompleted(BulkItemResponse translatedResponse) {
+        assertInvariants(ItemProcessingState.EXECUTED);
+        assert executionResult != null && translatedResponse.getItemId() == executionResult.getItemId();
+        assert translatedResponse.getItemId() == getCurrentItem().id();
+
+        if (translatedResponse.isFailed() == false && requestToExecute != null && requestToExecute != getCurrent())  {
+            request.items()[currentIndex] = new BulkItemRequest(request.items()[currentIndex].id(), requestToExecute);
+        }
+        getCurrentItem().setPrimaryResponse(translatedResponse);
+        currentItemState = ItemProcessingState.COMPLETED;
+        advance();
+    }
+
+    /** builds the bulk shard response to return to the user */
+    public BulkShardResponse buildShardResponse() {
+        assert hasMoreOperationsToExecute() == false;
+        return new BulkShardResponse(request.shardId(),
+            Arrays.stream(request.items()).map(BulkItemRequest::getPrimaryResponse).toArray(BulkItemResponse[]::new));
+    }
+
+    private boolean assertInvariants(ItemProcessingState... expectedCurrentState) {
+        assert Arrays.asList(expectedCurrentState).contains(currentItemState):
+            "expected current state [" + currentItemState + "] to be one of " + Arrays.toString(expectedCurrentState);
+        assert currentIndex >= 0 : currentIndex;
+        assert retryCounter >= 0 : retryCounter;
+        switch (currentItemState) {
+            case INITIAL:
+                assert requestToExecute == null : requestToExecute;
+                assert executionResult == null : executionResult;
+                break;
+            case TRANSLATED:
+                assert requestToExecute != null;
+                assert executionResult == null : executionResult;
+                break;
+            case WAIT_FOR_MAPPING_UPDATE:
+                assert requestToExecute == null;
+                assert executionResult == null : executionResult;
+                break;
+            case IMMEDIATE_RETRY:
+                assert requestToExecute != null;
+                assert executionResult == null : executionResult;
+                break;
+            case EXECUTED:
+                // requestToExecute can be null if the update ended up as NOOP
+                assert executionResult != null;
+                break;
+            case COMPLETED:
+                assert requestToExecute != null;
+                assert executionResult != null;
+                assert getCurrentItem().getPrimaryResponse() != null;
+                break;
+        }
+        return true;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
index 989172b711a13..6698aa4b62ab5 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
@@ -523,22 +523,6 @@ private int findNextMarker(byte marker, int from, BytesReference data, int lengt
         return -1;
     }
 
-    /**
-     * @return Whether this bulk request contains index request with an ingest pipeline enabled.
-     */
-    public boolean hasIndexRequestsWithPipelines() {
-        for (DocWriteRequest<?> actionRequest : requests) {
-            if (actionRequest instanceof IndexRequest) {
-                IndexRequest indexRequest = (IndexRequest) actionRequest;
-                if (Strings.hasText(indexRequest.getPipeline())) {
-                    return true;
-                }
-            }
-        }
-
-        return false;
-    }
-
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index 939b0b7024904..e3e94e8233944 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -47,6 +47,7 @@
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -54,6 +55,7 @@
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndexClosedException;
@@ -125,15 +127,6 @@ public TransportBulkAction(Settings settings, ThreadPool threadPool, TransportSe
 
     @Override
     protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener<BulkResponse> listener) {
-        if (bulkRequest.hasIndexRequestsWithPipelines()) {
-            if (clusterService.localNode().isIngestNode()) {
-                processBulkIndexIngestRequest(task, bulkRequest, listener);
-            } else {
-                ingestForwarder.forwardIngestRequest(BulkAction.INSTANCE, bulkRequest, listener);
-            }
-            return;
-        }
-
         final long startTime = relativeTime();
         final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
 
@@ -167,7 +160,7 @@ protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener<Bulk
             }
             // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back.
             if (autoCreateIndices.isEmpty()) {
-                executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
+                executeIngestAndBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
             } else {
                 final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size());
                 for (String index : autoCreateIndices) {
@@ -175,7 +168,7 @@ protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener<Bulk
                         @Override
                         public void onResponse(CreateIndexResponse result) {
                             if (counter.decrementAndGet() == 0) {
-                                executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
+                                executeIngestAndBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
                             }
                         }
 
@@ -191,7 +184,7 @@ public void onFailure(Exception e) {
                                 }
                             }
                             if (counter.decrementAndGet() == 0) {
-                                executeBulk(task, bulkRequest, startTime, ActionListener.wrap(listener::onResponse, inner -> {
+                                executeIngestAndBulk(task, bulkRequest, startTime, ActionListener.wrap(listener::onResponse, inner -> {
                                     inner.addSuppressed(e);
                                     listener.onFailure(inner);
                                 }), responses, indicesThatCannotBeCreated);
@@ -201,7 +194,47 @@ public void onFailure(Exception e) {
                 }
             }
         } else {
-            executeBulk(task, bulkRequest, startTime, listener, responses, emptyMap());
+            executeIngestAndBulk(task, bulkRequest, startTime, listener, responses, emptyMap());
+        }
+    }
+
+    private void executeIngestAndBulk(Task task, final BulkRequest bulkRequest, final long startTimeNanos,
+        final ActionListener<BulkResponse> listener, final AtomicArray<BulkItemResponse> responses,
+        Map<String, IndexNotFoundException> indicesThatCannotBeCreated) {
+        boolean hasIndexRequestsWithPipelines = false;
+        ImmutableOpenMap<String, IndexMetaData> indicesMetaData = clusterService.state().getMetaData().indices();
+        for (DocWriteRequest<?> actionRequest : bulkRequest.requests) {
+            if (actionRequest instanceof IndexRequest) {
+                IndexRequest indexRequest = (IndexRequest) actionRequest;
+                String pipeline = indexRequest.getPipeline();
+                if (pipeline == null) {
+                    IndexMetaData indexMetaData = indicesMetaData.get(indexRequest.index());
+                    if (indexMetaData == null) {
+                        indexRequest.setPipeline(IngestService.NOOP_PIPELINE_NAME);
+                    } else {
+                        String defaultPipeline = IndexSettings.DEFAULT_PIPELINE.get(indexMetaData.getSettings());
+                        indexRequest.setPipeline(defaultPipeline);
+                        if (IngestService.NOOP_PIPELINE_NAME.equals(defaultPipeline) == false) {
+                            hasIndexRequestsWithPipelines = true;
+                        }
+                    }
+                } else if (IngestService.NOOP_PIPELINE_NAME.equals(pipeline) == false) {
+                    hasIndexRequestsWithPipelines = true;
+                }
+            }
+        }
+        if (hasIndexRequestsWithPipelines) {
+            try {
+                if (clusterService.localNode().isIngestNode()) {
+                    processBulkIndexIngestRequest(task, bulkRequest, listener);
+                } else {
+                    ingestForwarder.forwardIngestRequest(BulkAction.INSTANCE, bulkRequest, listener);
+                }
+            } catch (Exception e) {
+                listener.onFailure(e);
+            }
+        } else {
+            executeBulk(task, bulkRequest, startTimeNanos, listener, responses, indicesThatCannotBeCreated);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index 15a98077eac4a..9c134ba4012da 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -29,30 +29,34 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.replication.ReplicationOperation;
-import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.action.support.replication.TransportWriteAction;
 import org.elasticsearch.action.update.UpdateHelper;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.index.mapper.MapperException;
 import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.seqno.SequenceNumbers;
@@ -60,12 +64,14 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
 
@@ -108,174 +114,167 @@ protected boolean resolveIndex() {
     }
 
     @Override
-    public WritePrimaryResult<BulkShardRequest, BulkShardResponse> shardOperationOnPrimary(
-            BulkShardRequest request, IndexShard primary) throws Exception {
-        return performOnPrimary(request, primary, updateHelper, threadPool::absoluteTimeInMillis, new ConcreteMappingUpdatePerformer());
+    protected WritePrimaryResult<BulkShardRequest, BulkShardResponse> shardOperationOnPrimary(BulkShardRequest request, IndexShard primary)
+        throws Exception {
+        ClusterStateObserver observer = new ClusterStateObserver(clusterService, request.timeout(), logger, threadPool.getThreadContext());
+        CheckedRunnable<Exception> waitForMappingUpdate = () -> {
+            PlainActionFuture<Void> waitingFuture = new PlainActionFuture<>();
+            observer.waitForNextChange(new ClusterStateObserver.Listener() {
+                @Override
+                public void onNewClusterState(ClusterState state) {
+                    waitingFuture.onResponse(null);
+                }
+
+                @Override
+                public void onClusterServiceClose() {
+                    waitingFuture.onFailure(new NodeClosedException(clusterService.localNode()));
+                }
+
+                @Override
+                public void onTimeout(TimeValue timeout) {
+                    waitingFuture.onFailure(
+                        new MapperException("timed out while waiting for a dynamic mapping update"));
+                }
+            });
+            waitingFuture.get();
+        };
+        return performOnPrimary(request, primary, updateHelper, threadPool::absoluteTimeInMillis,
+            new ConcreteMappingUpdatePerformer(), waitForMappingUpdate);
     }
 
     public static WritePrimaryResult<BulkShardRequest, BulkShardResponse> performOnPrimary(
-            BulkShardRequest request,
-            IndexShard primary,
-            UpdateHelper updateHelper,
-            LongSupplier nowInMillisSupplier,
-            MappingUpdatePerformer mappingUpdater) throws Exception {
-        final IndexMetaData metaData = primary.indexSettings().getIndexMetaData();
-        Translog.Location location = null;
-        for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) {
-            if (isAborted(request.items()[requestIndex].getPrimaryResponse()) == false) {
-                location = executeBulkItemRequest(metaData, primary, request, location, requestIndex,
-                    updateHelper, nowInMillisSupplier, mappingUpdater);
-            }
-        }
-        BulkItemResponse[] responses = new BulkItemResponse[request.items().length];
-        BulkItemRequest[] items = request.items();
-        for (int i = 0; i < items.length; i++) {
-            responses[i] = items[i].getPrimaryResponse();
-        }
-        BulkShardResponse response = new BulkShardResponse(request.shardId(), responses);
-        return new WritePrimaryResult<>(request, response, location, null, primary, logger);
+        BulkShardRequest request,
+        IndexShard primary,
+        UpdateHelper updateHelper,
+        LongSupplier nowInMillisSupplier,
+        MappingUpdatePerformer mappingUpdater,
+        CheckedRunnable<Exception> waitForMappingUpdate) throws Exception {
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(request, primary);
+        return performOnPrimary(context, updateHelper, nowInMillisSupplier, mappingUpdater, waitForMappingUpdate);
     }
 
-    private static BulkItemResultHolder executeIndexRequest(final IndexRequest indexRequest,
-                                                            final BulkItemRequest bulkItemRequest,
-                                                            final IndexShard primary,
-                                                            final MappingUpdatePerformer mappingUpdater) throws Exception {
-        Engine.IndexResult indexResult = executeIndexRequestOnPrimary(indexRequest, primary, mappingUpdater);
-        switch (indexResult.getResultType()) {
-            case SUCCESS:
-                IndexResponse response = new IndexResponse(primary.shardId(), indexRequest.type(), indexRequest.id(),
-                    indexResult.getSeqNo(), primary.getPrimaryTerm(), indexResult.getVersion(), indexResult.isCreated());
-                return new BulkItemResultHolder(response, indexResult, bulkItemRequest);
-            case FAILURE:
-                return new BulkItemResultHolder(null, indexResult, bulkItemRequest);
-            default:
-                throw new AssertionError("unknown result type for " + indexRequest + ": " + indexResult.getResultType());
+    private static WritePrimaryResult<BulkShardRequest, BulkShardResponse> performOnPrimary(
+        BulkPrimaryExecutionContext context, UpdateHelper updateHelper, LongSupplier nowInMillisSupplier,
+        MappingUpdatePerformer mappingUpdater, CheckedRunnable<Exception> waitForMappingUpdate) throws Exception {
+
+        while (context.hasMoreOperationsToExecute()) {
+            executeBulkItemRequest(context, updateHelper, nowInMillisSupplier, mappingUpdater, waitForMappingUpdate);
+            assert context.isInitial(); // either completed and moved to next or reset
         }
+        return new WritePrimaryResult<>(context.getBulkShardRequest(), context.buildShardResponse(), context.getLocationToSync(),
+            null, context.getPrimary(), logger);
     }
 
-    private static BulkItemResultHolder executeDeleteRequest(final DeleteRequest deleteRequest,
-                                                             final BulkItemRequest bulkItemRequest,
-                                                             final IndexShard primary,
-                                                             final MappingUpdatePerformer mappingUpdater) throws Exception {
-        Engine.DeleteResult deleteResult = executeDeleteRequestOnPrimary(deleteRequest, primary, mappingUpdater);
-        switch (deleteResult.getResultType()) {
-            case SUCCESS:
-                DeleteResponse response = new DeleteResponse(primary.shardId(), deleteRequest.type(), deleteRequest.id(),
-                    deleteResult.getSeqNo(), primary.getPrimaryTerm(), deleteResult.getVersion(), deleteResult.isFound());
-                return new BulkItemResultHolder(response, deleteResult, bulkItemRequest);
-            case FAILURE:
-                return new BulkItemResultHolder(null, deleteResult, bulkItemRequest);
-            case MAPPING_UPDATE_REQUIRED:
-                throw new AssertionError("delete operation leaked a mapping update " + deleteRequest);
-            default:
-                throw new AssertionError("unknown result type for " + deleteRequest + ": " + deleteResult.getResultType());
+    /** Executes bulk item requests and handles request execution exceptions */
+    static void executeBulkItemRequest(BulkPrimaryExecutionContext context, UpdateHelper updateHelper, LongSupplier nowInMillisSupplier,
+                                       MappingUpdatePerformer mappingUpdater, CheckedRunnable<Exception> waitForMappingUpdate)
+        throws Exception {
+        final DocWriteRequest.OpType opType = context.getCurrent().opType();
+
+        final UpdateHelper.Result updateResult;
+        if (opType == DocWriteRequest.OpType.UPDATE) {
+            final UpdateRequest updateRequest = (UpdateRequest) context.getCurrent();
+            try {
+                updateResult = updateHelper.prepare(updateRequest, context.getPrimary(), nowInMillisSupplier);
+            } catch (Exception failure) {
+                // we may fail translating a update to index or delete operation
+                // we use index result to communicate failure while translating update request
+                final Engine.Result result = new Engine.IndexResult(failure, updateRequest.version(), SequenceNumbers.UNASSIGNED_SEQ_NO);
+                context.setRequestToExecute(updateRequest);
+                context.markOperationAsExecuted(result);
+                context.markAsCompleted(context.getExecutionResult());
+                return;
+            }
+            // execute translated update request
+            switch (updateResult.getResponseResult()) {
+                case CREATED:
+                case UPDATED:
+                    IndexRequest indexRequest = updateResult.action();
+                    IndexMetaData metaData = context.getPrimary().indexSettings().getIndexMetaData();
+                    MappingMetaData mappingMd = metaData.mappingOrDefault(indexRequest.type());
+                    indexRequest.process(metaData.getCreationVersion(), mappingMd, updateRequest.concreteIndex());
+                    context.setRequestToExecute(indexRequest);
+                    break;
+                case DELETED:
+                    context.setRequestToExecute(updateResult.action());
+                    break;
+                case NOOP:
+                    context.markOperationAsNoOp(updateResult.action());
+                    context.markAsCompleted(context.getExecutionResult());
+                    return;
+                default:
+                    throw new IllegalStateException("Illegal update operation " + updateResult.getResponseResult());
+            }
+        } else {
+            context.setRequestToExecute(context.getCurrent());
+            updateResult = null;
         }
-    }
 
-    static Translog.Location calculateTranslogLocation(final Translog.Location originalLocation,
-                                                       final BulkItemResultHolder bulkItemResult) {
-        final Engine.Result operationResult = bulkItemResult.operationResult;
-        if (operationResult != null && operationResult.getResultType() == Engine.Result.Type.SUCCESS) {
-            return locationToSync(originalLocation, operationResult.getTranslogLocation());
+        assert context.getRequestToExecute() != null; // also checks that we're in TRANSLATED state
+
+        if (context.getRequestToExecute().opType() == DocWriteRequest.OpType.DELETE) {
+            executeDeleteRequestOnPrimary(context, mappingUpdater);
         } else {
-            return originalLocation;
+            executeIndexRequestOnPrimary(context, mappingUpdater);
         }
+
+        if (context.requiresWaitingForMappingUpdate()) {
+            try {
+                waitForMappingUpdate.run();
+                context.resetForExecutionForRetry();
+            } catch (Exception e) {
+                context.failOnMappingUpdate(e);
+            }
+            return;
+        }
+
+        assert context.isOperationExecuted();
+
+        if (opType == DocWriteRequest.OpType.UPDATE &&
+            context.getExecutionResult().isFailed() &&
+            isConflictException(context.getExecutionResult().getFailure().getCause())) {
+            final UpdateRequest updateRequest = (UpdateRequest) context.getCurrent();
+            if (context.getRetryCounter() < updateRequest.retryOnConflict()) {
+                context.resetForExecutionForRetry();
+                return;
+            }
+        }
+
+        finalizePrimaryOperationOnCompletion(context, opType, updateResult);
     }
 
-    // Visible for unit testing
-    /**
-     * Creates a BulkItemResponse for the primary operation and returns it. If no bulk response is
-     * needed (because one already exists and the operation failed), then return null.
-     */
-    static BulkItemResponse createPrimaryResponse(BulkItemResultHolder bulkItemResult,
-                                                  final DocWriteRequest.OpType opType,
-                                                  BulkShardRequest request) {
-        final Engine.Result operationResult = bulkItemResult.operationResult;
-        final DocWriteResponse response = bulkItemResult.response;
-        final BulkItemRequest replicaRequest = bulkItemResult.replicaRequest;
-
-        if (operationResult == null) { // in case of noop update operation
-            assert response.getResult() == DocWriteResponse.Result.NOOP : "only noop updates can have a null operation";
-            return new BulkItemResponse(replicaRequest.id(), opType, response);
-
-        } else if (operationResult.getResultType() == Engine.Result.Type.SUCCESS) {
-            BulkItemResponse primaryResponse = new BulkItemResponse(replicaRequest.id(), opType, response);
-            // set a blank ShardInfo so we can safely send it to the replicas. We won't use it in the real response though.
-            primaryResponse.getResponse().setShardInfo(new ShardInfo());
-            return primaryResponse;
-
-        } else if (operationResult.getResultType() == Engine.Result.Type.FAILURE) {
-            DocWriteRequest<?> docWriteRequest = replicaRequest.request();
-            Exception failure = operationResult.getFailure();
-            if (isConflictException(failure)) {
+    private static void finalizePrimaryOperationOnCompletion(BulkPrimaryExecutionContext context, DocWriteRequest.OpType opType,
+                                                             UpdateHelper.Result updateResult) {
+        final BulkItemResponse executionResult = context.getExecutionResult();
+        if (opType == DocWriteRequest.OpType.UPDATE) {
+            final UpdateRequest updateRequest = (UpdateRequest) context.getCurrent();
+            context.markAsCompleted(
+                processUpdateResponse(updateRequest, context.getConcreteIndex(), executionResult, updateResult));
+        } else if (executionResult.isFailed()) {
+            final Exception failure = executionResult.getFailure().getCause();
+            final DocWriteRequest docWriteRequest = context.getCurrent();
+            if (TransportShardBulkAction.isConflictException(failure)) {
                 logger.trace(() -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}",
-                    request.shardId(), docWriteRequest.opType().getLowercase(), request), failure);
+                    context.getPrimary().shardId(), docWriteRequest.opType().getLowercase(), docWriteRequest), failure);
             } else {
                 logger.debug(() -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}",
-                    request.shardId(), docWriteRequest.opType().getLowercase(), request), failure);
+                    context.getPrimary().shardId(), docWriteRequest.opType().getLowercase(), docWriteRequest), failure);
             }
 
+            final BulkItemResponse primaryResponse;
             // if it's a conflict failure, and we already executed the request on a primary (and we execute it
             // again, due to primary relocation and only processing up to N bulk items when the shard gets closed)
             // then just use the response we got from the failed execution
-            if (replicaRequest.getPrimaryResponse() == null || isConflictException(failure) == false) {
-                return new BulkItemResponse(replicaRequest.id(), docWriteRequest.opType(),
-                        // Make sure to use request.index() here, if you
-                        // use docWriteRequest.index() it will use the
-                        // concrete index instead of an alias if used!
-                        new BulkItemResponse.Failure(request.index(), docWriteRequest.type(), docWriteRequest.id(),
-                                failure, operationResult.getSeqNo()));
+            if (TransportShardBulkAction.isConflictException(failure) && context.getPreviousPrimaryResponse() != null) {
+                primaryResponse = context.getPreviousPrimaryResponse();
             } else {
-                assert replicaRequest.getPrimaryResponse() != null : "replica request must have a primary response";
-                return null;
+                primaryResponse = executionResult;
             }
+            context.markAsCompleted(primaryResponse);
         } else {
-            throw new AssertionError("unknown result type for " + request + ": " + operationResult.getResultType());
+            context.markAsCompleted(executionResult);
         }
-    }
-
-    /** Executes bulk item requests and handles request execution exceptions */
-    static Translog.Location executeBulkItemRequest(IndexMetaData metaData, IndexShard primary,
-                                                    BulkShardRequest request, Translog.Location location,
-                                                    int requestIndex, UpdateHelper updateHelper,
-                                                    LongSupplier nowInMillisSupplier,
-                                                    final MappingUpdatePerformer mappingUpdater) throws Exception {
-        final DocWriteRequest<?> itemRequest = request.items()[requestIndex].request();
-        final DocWriteRequest.OpType opType = itemRequest.opType();
-        final BulkItemResultHolder responseHolder;
-        switch (itemRequest.opType()) {
-            case CREATE:
-            case INDEX:
-                responseHolder = executeIndexRequest((IndexRequest) itemRequest,
-                        request.items()[requestIndex], primary, mappingUpdater);
-                break;
-            case UPDATE:
-                responseHolder = executeUpdateRequest((UpdateRequest) itemRequest, primary, metaData, request,
-                        requestIndex, updateHelper, nowInMillisSupplier, mappingUpdater);
-                break;
-            case DELETE:
-                responseHolder = executeDeleteRequest((DeleteRequest) itemRequest, request.items()[requestIndex], primary, mappingUpdater);
-                break;
-            default: throw new IllegalStateException("unexpected opType [" + itemRequest.opType() + "] found");
-        }
-
-        final BulkItemRequest replicaRequest = responseHolder.replicaRequest;
-
-        // update the bulk item request because update request execution can mutate the bulk item request
-        request.items()[requestIndex] = replicaRequest;
-
-        // Retrieve the primary response, and update the replica request with the primary's response
-        BulkItemResponse primaryResponse = createPrimaryResponse(responseHolder, opType, request);
-        if (primaryResponse != null) {
-            replicaRequest.setPrimaryResponse(primaryResponse);
-        }
-
-        // Update the translog with the new location, if needed
-        return calculateTranslogLocation(location, responseHolder);
-    }
-
-    private static boolean isAborted(BulkItemResponse response) {
-        return response != null && response.isFailed() && response.getFailure().isAborted();
+        assert context.isInitial();
     }
 
     private static boolean isConflictException(final Exception e) {
@@ -285,150 +284,50 @@ private static boolean isConflictException(final Exception e) {
     /**
      * Creates a new bulk item result from the given requests and result of performing the update operation on the shard.
      */
-    static BulkItemResultHolder processUpdateResponse(final UpdateRequest updateRequest, final String concreteIndex,
-                                                      final Engine.Result result, final UpdateHelper.Result translate,
-                                                      final IndexShard primary, final int bulkReqId) {
-        assert result.getSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "failed result should not have a sequence number";
-
-        Engine.Operation.TYPE opType = result.getOperationType();
-
-        final UpdateResponse updateResponse;
-        final BulkItemRequest replicaRequest;
-
-        // enrich update response and set translated update (index/delete) request for replica execution in bulk items
-        if (opType == Engine.Operation.TYPE.INDEX) {
-            assert result instanceof Engine.IndexResult : result.getClass();
-            final IndexRequest updateIndexRequest = translate.action();
-            final IndexResponse indexResponse = new IndexResponse(primary.shardId(), updateIndexRequest.type(), updateIndexRequest.id(),
-                    result.getSeqNo(), primary.getPrimaryTerm(), result.getVersion(), ((Engine.IndexResult) result).isCreated());
-            updateResponse = new UpdateResponse(indexResponse.getShardInfo(), indexResponse.getShardId(), indexResponse.getType(),
-                    indexResponse.getId(), indexResponse.getSeqNo(), indexResponse.getPrimaryTerm(), indexResponse.getVersion(),
-                    indexResponse.getResult());
-
-            if (updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) {
-                final BytesReference indexSourceAsBytes = updateIndexRequest.source();
-                final Tuple<XContentType, Map<String, Object>> sourceAndContent =
-                        XContentHelper.convertToMap(indexSourceAsBytes, true, updateIndexRequest.getContentType());
-                updateResponse.setGetResult(UpdateHelper.extractGetResult(updateRequest, concreteIndex,
-                                indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes));
-            }
-            // set translated request as replica request
-            replicaRequest = new BulkItemRequest(bulkReqId, updateIndexRequest);
-
-        } else if (opType == Engine.Operation.TYPE.DELETE) {
-            assert result instanceof Engine.DeleteResult : result.getClass();
-            final DeleteRequest updateDeleteRequest = translate.action();
-
-            final DeleteResponse deleteResponse = new DeleteResponse(primary.shardId(), updateDeleteRequest.type(), updateDeleteRequest.id(),
-                    result.getSeqNo(), primary.getPrimaryTerm(), result.getVersion(), ((Engine.DeleteResult) result).isFound());
+    static BulkItemResponse processUpdateResponse(final UpdateRequest updateRequest, final String concreteIndex,
+                                                  BulkItemResponse operationResponse,
+                                                  final UpdateHelper.Result translate) {
+
+        final BulkItemResponse response;
+        DocWriteResponse.Result translatedResult = translate.getResponseResult();
+        if (operationResponse.isFailed()) {
+            response = new BulkItemResponse(operationResponse.getItemId(), DocWriteRequest.OpType.UPDATE, operationResponse.getFailure());
+        } else {
 
-            updateResponse = new UpdateResponse(deleteResponse.getShardInfo(), deleteResponse.getShardId(),
+            final UpdateResponse updateResponse;
+            if (translatedResult == DocWriteResponse.Result.CREATED || translatedResult == DocWriteResponse.Result.UPDATED) {
+                final IndexRequest updateIndexRequest = translate.action();
+                final IndexResponse indexResponse = operationResponse.getResponse();
+                updateResponse = new UpdateResponse(indexResponse.getShardInfo(), indexResponse.getShardId(),
+                    indexResponse.getType(), indexResponse.getId(), indexResponse.getSeqNo(), indexResponse.getPrimaryTerm(),
+                    indexResponse.getVersion(), indexResponse.getResult());
+
+                if (updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) {
+                    final BytesReference indexSourceAsBytes = updateIndexRequest.source();
+                    final Tuple<XContentType, Map<String, Object>> sourceAndContent =
+                        XContentHelper.convertToMap(indexSourceAsBytes, true, updateIndexRequest.getContentType());
+                    updateResponse.setGetResult(UpdateHelper.extractGetResult(updateRequest, concreteIndex,
+                        indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes));
+                }
+            } else if (translatedResult == DocWriteResponse.Result.DELETED) {
+                final DeleteResponse deleteResponse = operationResponse.getResponse();
+                updateResponse = new UpdateResponse(deleteResponse.getShardInfo(), deleteResponse.getShardId(),
                     deleteResponse.getType(), deleteResponse.getId(), deleteResponse.getSeqNo(), deleteResponse.getPrimaryTerm(),
                     deleteResponse.getVersion(), deleteResponse.getResult());
 
-            final GetResult getResult = UpdateHelper.extractGetResult(updateRequest, concreteIndex, deleteResponse.getVersion(),
+                final GetResult getResult = UpdateHelper.extractGetResult(updateRequest, concreteIndex, deleteResponse.getVersion(),
                     translate.updatedSourceAsMap(), translate.updateSourceContentType(), null);
 
-            updateResponse.setGetResult(getResult);
-            // set translated request as replica request
-            replicaRequest = new BulkItemRequest(bulkReqId, updateDeleteRequest);
-
-        } else {
-            throw new IllegalArgumentException("unknown operation type: " + opType);
-        }
-
-        return new BulkItemResultHolder(updateResponse, result, replicaRequest);
-    }
-
-    /**
-     * Executes update request once, delegating to a index or delete operation after translation.
-     * NOOP updates are indicated by returning a <code>null</code> operation in {@link BulkItemResultHolder}
-     */
-    static BulkItemResultHolder executeUpdateRequestOnce(UpdateRequest updateRequest, IndexShard primary,
-                                                         IndexMetaData metaData, String concreteIndex,
-                                                         UpdateHelper updateHelper, LongSupplier nowInMillis,
-                                                         BulkItemRequest primaryItemRequest, int bulkReqId,
-                                                         final MappingUpdatePerformer mappingUpdater) throws Exception {
-        final UpdateHelper.Result translate;
-        // translate update request
-        try {
-            translate = updateHelper.prepare(updateRequest, primary, nowInMillis);
-        } catch (Exception failure) {
-            // we may fail translating a update to index or delete operation
-            // we use index result to communicate failure while translating update request
-            final Engine.Result result = new Engine.IndexResult(failure, updateRequest.version(), SequenceNumbers.UNASSIGNED_SEQ_NO);
-            return new BulkItemResultHolder(null, result, primaryItemRequest);
-        }
-
-        final Engine.Result result;
-        // execute translated update request
-        switch (translate.getResponseResult()) {
-            case CREATED:
-            case UPDATED:
-                IndexRequest indexRequest = translate.action();
-                MappingMetaData mappingMd = metaData.mappingOrDefault(indexRequest.type());
-                indexRequest.process(metaData.getCreationVersion(), mappingMd, concreteIndex);
-                result = executeIndexRequestOnPrimary(indexRequest, primary, mappingUpdater);
-                break;
-            case DELETED:
-                DeleteRequest deleteRequest = translate.action();
-                result = executeDeleteRequestOnPrimary(deleteRequest, primary, mappingUpdater);
-                break;
-            case NOOP:
-                primary.noopUpdate(updateRequest.type());
-                result = null;
-                break;
-            default: throw new IllegalStateException("Illegal update operation " + translate.getResponseResult());
-        }
-
-        if (result == null) {
-            // this is a noop operation
-            final UpdateResponse updateResponse = translate.action();
-            return new BulkItemResultHolder(updateResponse, result, primaryItemRequest);
-        } else if (result.getResultType() == Engine.Result.Type.FAILURE) {
-            // There was a result, and the result was a failure
-            return new BulkItemResultHolder(null, result, primaryItemRequest);
-        } else if (result.getResultType() == Engine.Result.Type.SUCCESS) {
-            // It was successful, we need to construct the response and return it
-            return processUpdateResponse(updateRequest, concreteIndex, result, translate, primary, bulkReqId);
-        } else {
-            throw new AssertionError("unknown result type for " + updateRequest + ": " + result.getResultType());
-        }
-    }
-
-    /**
-     * Executes update request, delegating to a index or delete operation after translation,
-     * handles retries on version conflict and constructs update response
-     * NOOP updates are indicated by returning a <code>null</code> operation
-     * in {@link BulkItemResultHolder}
-     */
-    private static BulkItemResultHolder executeUpdateRequest(UpdateRequest updateRequest, IndexShard primary,
-                                                             IndexMetaData metaData, BulkShardRequest request,
-                                                             int requestIndex, UpdateHelper updateHelper,
-                                                             LongSupplier nowInMillis,
-                                                             final MappingUpdatePerformer mappingUpdater) throws Exception {
-        BulkItemRequest primaryItemRequest = request.items()[requestIndex];
-        assert primaryItemRequest.request() == updateRequest
-                : "expected bulk item request to contain the original update request, got: " +
-                primaryItemRequest.request() + " and " + updateRequest;
-
-        BulkItemResultHolder holder = null;
-        // There must be at least one attempt
-        int maxAttempts = Math.max(1, updateRequest.retryOnConflict());
-        for (int attemptCount = 0; attemptCount < maxAttempts; attemptCount++) {
-
-            holder = executeUpdateRequestOnce(updateRequest, primary, metaData, request.index(), updateHelper,
-                    nowInMillis, primaryItemRequest, request.items()[requestIndex].id(), mappingUpdater);
-
-            // It was either a successful request, or it was a non-conflict failure
-            if (holder.isVersionConflict() == false) {
-                return holder;
+                updateResponse.setGetResult(getResult);
+            } else {
+                throw new IllegalArgumentException("unknown operation type: " + translatedResult);
             }
+            response = new BulkItemResponse(operationResponse.getItemId(), DocWriteRequest.OpType.UPDATE, updateResponse);
         }
-        // We ran out of tries and haven't returned a valid bulk item response, so return the last one generated
-        return holder;
+        return response;
     }
 
+
     /** Modes for executing item request on replica depending on corresponding primary execution result */
     public enum ReplicaItemExecutionMode {
 
@@ -451,6 +350,7 @@ public enum ReplicaItemExecutionMode {
 
     /**
      * Determines whether a bulk item request should be executed on the replica.
+     *
      * @return {@link ReplicaItemExecutionMode#NORMAL} upon normal primary execution with no failures
      * {@link ReplicaItemExecutionMode#FAILURE} upon primary execution failure after sequence no generation
      * {@link ReplicaItemExecutionMode#NOOP} upon primary execution failure before sequence no generation or
@@ -461,8 +361,8 @@ static ReplicaItemExecutionMode replicaItemExecutionMode(final BulkItemRequest r
         assert primaryResponse != null : "expected primary response to be set for item [" + index + "] request [" + request.request() + "]";
         if (primaryResponse.isFailed()) {
             return primaryResponse.getFailure().getSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO
-                    ? ReplicaItemExecutionMode.FAILURE // we have a seq no generated with the failure, replicate as no-op
-                    : ReplicaItemExecutionMode.NOOP; // no seq no generated, ignore replication
+                ? ReplicaItemExecutionMode.FAILURE // we have a seq no generated with the failure, replicate as no-op
+                : ReplicaItemExecutionMode.NOOP; // no seq no generated, ignore replication
         } else {
             // TODO: once we know for sure that every operation that has been processed on the primary is assigned a seq#
             // (i.e., all nodes on the cluster are on v6.0.0 or higher) we can use the existence of a seq# to indicate whether
@@ -470,8 +370,8 @@ static ReplicaItemExecutionMode replicaItemExecutionMode(final BulkItemRequest r
             // ReplicaItemExecutionMode enum and have a simple boolean check for seq != UNASSIGNED_SEQ_NO which will work for
             // both failures and indexing operations.
             return primaryResponse.getResponse().getResult() != DocWriteResponse.Result.NOOP
-                    ? ReplicaItemExecutionMode.NORMAL // execution successful on primary
-                    : ReplicaItemExecutionMode.NOOP; // ignore replication
+                ? ReplicaItemExecutionMode.NORMAL // execution successful on primary
+                : ReplicaItemExecutionMode.NOOP; // ignore replication
         }
     }
 
@@ -527,7 +427,7 @@ private static Engine.Result performOpOnReplica(DocWriteResponse primaryResponse
                 break;
             case DELETE:
                 DeleteRequest deleteRequest = (DeleteRequest) docWriteRequest;
-                result =  replica.applyDeleteOperationOnReplica(primaryResponse.getSeqNo(), primaryResponse.getVersion(),
+                result = replica.applyDeleteOperationOnReplica(primaryResponse.getSeqNo(), primaryResponse.getVersion(),
                     deleteRequest.type(), deleteRequest.id());
                 break;
             default:
@@ -550,56 +450,62 @@ private static Engine.Result performOpOnReplica(DocWriteResponse primaryResponse
     }
 
     /** Executes index operation on primary shard after updates mapping if dynamic mappings are found */
-    static Engine.IndexResult executeIndexRequestOnPrimary(IndexRequest request, IndexShard primary,
-                                                           MappingUpdatePerformer mappingUpdater) throws Exception {
+    private static void executeIndexRequestOnPrimary(BulkPrimaryExecutionContext context,
+                                             MappingUpdatePerformer mappingUpdater) throws Exception {
+        final IndexRequest request = context.getRequestToExecute();
+        final IndexShard primary = context.getPrimary();
         final SourceToParse sourceToParse =
             SourceToParse.source(request.index(), request.type(), request.id(), request.source(), request.getContentType())
                 .routing(request.routing());
-        return executeOnPrimaryWhileHandlingMappingUpdates(primary.shardId(), request.type(),
+        executeOnPrimaryWhileHandlingMappingUpdates(context,
             () ->
                 primary.applyIndexOperationOnPrimary(request.version(), request.versionType(), sourceToParse,
                     request.getAutoGeneratedTimestamp(), request.isRetry()),
-            e -> new Engine.IndexResult(e, request.version()),
-            mappingUpdater);
+            e -> primary.getFailedIndexResult(e, request.version()),
+            context::markOperationAsExecuted,
+            mapping -> mappingUpdater.updateMappings(mapping, primary.shardId(), request.type()));
     }
 
-    private static Engine.DeleteResult executeDeleteRequestOnPrimary(DeleteRequest request, IndexShard primary,
-                                                                     MappingUpdatePerformer mappingUpdater) throws Exception {
-        return executeOnPrimaryWhileHandlingMappingUpdates(primary.shardId(), request.type(),
+    private static void executeDeleteRequestOnPrimary(BulkPrimaryExecutionContext context,
+                                                      MappingUpdatePerformer mappingUpdater) throws Exception {
+        final DeleteRequest request = context.getRequestToExecute();
+        final IndexShard primary = context.getPrimary();
+        executeOnPrimaryWhileHandlingMappingUpdates(context,
             () -> primary.applyDeleteOperationOnPrimary(request.version(), request.type(), request.id(), request.versionType()),
-            e -> new Engine.DeleteResult(e, request.version()),
-            mappingUpdater);
+            e -> primary.getFailedDeleteResult(e, request.version()),
+            context::markOperationAsExecuted,
+            mapping -> mappingUpdater.updateMappings(mapping, primary.shardId(), request.type()));
     }
 
-    private static <T extends Engine.Result> T executeOnPrimaryWhileHandlingMappingUpdates(ShardId shardId, String type,
-                                                                                           CheckedSupplier<T, IOException> toExecute,
-                                                                                           Function<Exception, T> onError,
-                                                                                           MappingUpdatePerformer mappingUpdater)
+    private static <T extends Engine.Result> void executeOnPrimaryWhileHandlingMappingUpdates(
+        BulkPrimaryExecutionContext context, CheckedSupplier<T, IOException> toExecute,
+        Function<Exception, T> exceptionToResult, Consumer<T> onComplete, Consumer<Mapping> mappingUpdater)
         throws IOException {
         T result = toExecute.get();
         if (result.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
             // try to update the mappings and try again.
             try {
-                mappingUpdater.updateMappings(result.getRequiredMappingUpdate(), shardId, type);
+                mappingUpdater.accept(result.getRequiredMappingUpdate());
             } catch (Exception e) {
                 // failure to update the mapping should translate to a failure of specific requests. Other requests
                 // still need to be executed and replicated.
-                return onError.apply(e);
+                onComplete.accept(exceptionToResult.apply(e));
+                return;
             }
 
+            // TODO - we can fall back to a wait for cluster state update but I'm keeping the logic the same for now
             result = toExecute.get();
 
             if (result.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
                 // double mapping update. We assume that the successful mapping update wasn't yet processed on the node
                 // and retry the entire request again.
-                throw new ReplicationOperation.RetryOnPrimaryException(shardId,
-                    "Dynamic mappings are not available on the node that holds the primary yet");
+                context.markAsRequiringMappingUpdate();
+            } else {
+                onComplete.accept(result);
             }
+        } else {
+            onComplete.accept(result);
         }
-        assert result.getFailure() instanceof ReplicationOperation.RetryOnPrimaryException == false :
-            "IndexShard shouldn't use RetryOnPrimaryException. got " + result.getFailure();
-        return result;
-
     }
 
     class ConcreteMappingUpdatePerformer implements MappingUpdatePerformer {
diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index 57e8ea6613817..339880dad44bf 100644
--- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -185,6 +185,10 @@ public ActionRequestValidationException validate() {
             validationException = addValidationError("an id must be provided if version type or value are set", validationException);
         }
 
+        if (pipeline != null && pipeline.isEmpty()) {
+            validationException = addValidationError("pipeline cannot be an empty string", validationException);
+        }
+
         return validationException;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineAction.java b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineAction.java
index e0df57a6dadf6..902614a3e83b3 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineAction.java
@@ -20,8 +20,9 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 
-public class DeletePipelineAction extends Action<WritePipelineResponse> {
+public class DeletePipelineAction extends Action<AcknowledgedResponse> {
 
     public static final DeletePipelineAction INSTANCE = new DeletePipelineAction();
     public static final String NAME = "cluster:admin/ingest/pipeline/delete";
@@ -31,7 +32,7 @@ public DeletePipelineAction() {
     }
 
     @Override
-    public WritePipelineResponse newResponse() {
-        return new WritePipelineResponse();
+    public AcknowledgedResponse newResponse() {
+        return new AcknowledgedResponse();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java
index afe4ea1b5907b..b08e0cb247205 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java
@@ -20,9 +20,10 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ElasticsearchClient;
 
-public class DeletePipelineRequestBuilder extends ActionRequestBuilder<DeletePipelineRequest, WritePipelineResponse> {
+public class DeletePipelineRequestBuilder extends ActionRequestBuilder<DeletePipelineRequest, AcknowledgedResponse> {
 
     public DeletePipelineRequestBuilder(ElasticsearchClient client, DeletePipelineAction action) {
         super(client, action, new DeletePipelineRequest());
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
index 45cb83634f84f..d3cd052ecad1e 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -34,7 +35,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-public class DeletePipelineTransportAction extends TransportMasterNodeAction<DeletePipelineRequest, WritePipelineResponse> {
+public class DeletePipelineTransportAction extends TransportMasterNodeAction<DeletePipelineRequest, AcknowledgedResponse> {
 
     private final PipelineStore pipelineStore;
     private final ClusterService clusterService;
@@ -54,12 +55,12 @@ protected String executor() {
     }
 
     @Override
-    protected WritePipelineResponse newResponse() {
-        return new WritePipelineResponse();
+    protected AcknowledgedResponse newResponse() {
+        return new AcknowledgedResponse();
     }
 
     @Override
-    protected void masterOperation(DeletePipelineRequest request, ClusterState state, ActionListener<WritePipelineResponse> listener) throws Exception {
+    protected void masterOperation(DeletePipelineRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) throws Exception {
         pipelineStore.delete(clusterService, request, listener);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineAction.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineAction.java
index c4784598ae75f..4ebcff127cccd 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineAction.java
@@ -20,8 +20,9 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 
-public class PutPipelineAction extends Action<WritePipelineResponse> {
+public class PutPipelineAction extends Action<AcknowledgedResponse> {
 
     public static final PutPipelineAction INSTANCE = new PutPipelineAction();
     public static final String NAME = "cluster:admin/ingest/pipeline/put";
@@ -31,7 +32,7 @@ public PutPipelineAction() {
     }
 
     @Override
-    public WritePipelineResponse newResponse() {
-        return new WritePipelineResponse();
+    public AcknowledgedResponse newResponse() {
+        return new AcknowledgedResponse();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java
index ffbb94d27a05b..1919d98c7e1f9 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java
@@ -20,11 +20,12 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
 
-public class PutPipelineRequestBuilder extends ActionRequestBuilder<PutPipelineRequest, WritePipelineResponse> {
+public class PutPipelineRequestBuilder extends ActionRequestBuilder<PutPipelineRequest, AcknowledgedResponse> {
 
     public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action) {
         super(client, action, new PutPipelineRequest());
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
index 17af73c167704..abe8f49272c77 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
@@ -43,7 +44,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-public class PutPipelineTransportAction extends TransportMasterNodeAction<PutPipelineRequest, WritePipelineResponse> {
+public class PutPipelineTransportAction extends TransportMasterNodeAction<PutPipelineRequest, AcknowledgedResponse> {
 
     private final PipelineStore pipelineStore;
     private final ClusterService clusterService;
@@ -66,12 +67,12 @@ protected String executor() {
     }
 
     @Override
-    protected WritePipelineResponse newResponse() {
-        return new WritePipelineResponse();
+    protected AcknowledgedResponse newResponse() {
+        return new AcknowledgedResponse();
     }
 
     @Override
-    protected void masterOperation(PutPipelineRequest request, ClusterState state, ActionListener<WritePipelineResponse> listener) throws Exception {
+    protected void masterOperation(PutPipelineRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) throws Exception {
         NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
         nodesInfoRequest.clear();
         nodesInfoRequest.ingest(true);
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
index c6e0b21dffd5d..3d3737b0638cd 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
@@ -138,8 +138,7 @@ protected void metadataToXContent(XContentBuilder builder, Params params) throws
         builder.field("grouped", group); // notify that it's grouped
         builder.field("failed_shards");
         builder.startArray();
-        ShardOperationFailedException[] failures = params.paramAsBoolean("group_shard_failures", true) ?
-                ExceptionsHelper.groupBy(shardFailures) : shardFailures;
+        ShardOperationFailedException[] failures = group ? ExceptionsHelper.groupBy(shardFailures) : shardFailures;
         for (ShardOperationFailedException failure : failures) {
             builder.startObject();
             failure.toXContent(builder, params);
diff --git a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
index f2ba62fefd43f..98418153d501a 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchException;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.transport.RemoteClusterAware;
 
 import java.io.IOException;
 
@@ -42,7 +43,7 @@
 /**
  * Represents a failure to search on a specific shard.
  */
-public class ShardSearchFailure implements ShardOperationFailedException {
+public class ShardSearchFailure extends ShardOperationFailedException {
 
     private static final String REASON_FIELD = "reason";
     private static final String NODE_FIELD = "node";
@@ -52,9 +53,6 @@ public class ShardSearchFailure implements ShardOperationFailedException {
     public static final ShardSearchFailure[] EMPTY_ARRAY = new ShardSearchFailure[0];
 
     private SearchShardTarget shardTarget;
-    private String reason;
-    private RestStatus status;
-    private Throwable cause;
 
     private ShardSearchFailure() {
 
@@ -65,25 +63,18 @@ public ShardSearchFailure(Exception e) {
     }
 
     public ShardSearchFailure(Exception e, @Nullable SearchShardTarget shardTarget) {
+        super(shardTarget == null ? null : shardTarget.getFullyQualifiedIndexName(),
+            shardTarget == null ? -1 : shardTarget.getShardId().getId(),
+            ExceptionsHelper.detailedMessage(e),
+            ExceptionsHelper.status(ExceptionsHelper.unwrapCause(e)),
+            ExceptionsHelper.unwrapCause(e));
+
         final Throwable actual = ExceptionsHelper.unwrapCause(e);
-        if (actual != null && actual instanceof SearchException) {
+        if (actual instanceof SearchException) {
             this.shardTarget = ((SearchException) actual).shard();
         } else if (shardTarget != null) {
             this.shardTarget = shardTarget;
         }
-        status = ExceptionsHelper.status(actual);
-        this.reason = ExceptionsHelper.detailedMessage(e);
-        this.cause = actual;
-    }
-
-    public ShardSearchFailure(String reason, SearchShardTarget shardTarget) {
-        this(reason, shardTarget, RestStatus.INTERNAL_SERVER_ERROR);
-    }
-
-    private ShardSearchFailure(String reason, SearchShardTarget shardTarget, RestStatus status) {
-        this.shardTarget = shardTarget;
-        this.reason = reason;
-        this.status = status;
     }
 
     /**
@@ -94,41 +85,6 @@ public SearchShardTarget shard() {
         return this.shardTarget;
     }
 
-    @Override
-    public RestStatus status() {
-        return this.status;
-    }
-
-    /**
-     * The index the search failed on.
-     */
-    @Override
-    public String index() {
-        if (shardTarget != null) {
-            return shardTarget.getIndex();
-        }
-        return null;
-    }
-
-    /**
-     * The shard id the search failed on.
-     */
-    @Override
-    public int shardId() {
-        if (shardTarget != null) {
-            return shardTarget.getShardId().id();
-        }
-        return -1;
-    }
-
-    /**
-     * The reason of the failure.
-     */
-    @Override
-    public String reason() {
-        return this.reason;
-    }
-
     @Override
     public String toString() {
         return "shard [" + (shardTarget == null ? "_na" : shardTarget) + "], reason [" + reason + "], cause [" +
@@ -171,12 +127,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (shardTarget != null) {
             builder.field(NODE_FIELD, shardTarget.getNodeId());
         }
-        if (cause != null) {
-            builder.field(REASON_FIELD);
-            builder.startObject();
-            ElasticsearchException.generateThrowableXContent(builder, params, cause);
-            builder.endObject();
-        }
+        builder.field(REASON_FIELD);
+        builder.startObject();
+        ElasticsearchException.generateThrowableXContent(builder, params, cause);
+        builder.endObject();
         return builder;
     }
 
@@ -186,6 +140,7 @@ public static ShardSearchFailure fromXContent(XContentParser parser) throws IOEx
         String currentFieldName = null;
         int shardId = -1;
         String indexName = null;
+        String clusterAlias = null;
         String nodeId = null;
         ElasticsearchException exception = null;
         while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -196,6 +151,11 @@ public static ShardSearchFailure fromXContent(XContentParser parser) throws IOEx
                     shardId  = parser.intValue();
                 } else if (INDEX_FIELD.equals(currentFieldName)) {
                     indexName  = parser.text();
+                    int indexOf = indexName.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR);
+                    if (indexOf > 0) {
+                        clusterAlias = indexName.substring(0, indexOf);
+                        indexName = indexName.substring(indexOf + 1);
+                    }
                 } else if (NODE_FIELD.equals(currentFieldName)) {
                     nodeId  = parser.text();
                 } else {
@@ -214,13 +174,8 @@ public static ShardSearchFailure fromXContent(XContentParser parser) throws IOEx
         SearchShardTarget searchShardTarget = null;
         if (nodeId != null) {
             searchShardTarget = new SearchShardTarget(nodeId,
-                    new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), null, OriginalIndices.NONE);
+                    new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), clusterAlias, OriginalIndices.NONE);
         }
         return new ShardSearchFailure(exception, searchShardTarget);
     }
-
-    @Override
-    public Throwable getCause() {
-        return cause;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java b/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
index 8a4a787fbe5f2..d297df478a4b8 100644
--- a/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
+++ b/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
@@ -28,8 +28,6 @@
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -37,7 +35,7 @@
 import static org.elasticsearch.ExceptionsHelper.detailedMessage;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 
-public class DefaultShardOperationFailedException implements ShardOperationFailedException {
+public class DefaultShardOperationFailedException extends ShardOperationFailedException {
 
     private static final String INDEX = "index";
     private static final String SHARD_ID = "shard";
@@ -52,56 +50,16 @@ public class DefaultShardOperationFailedException implements ShardOperationFaile
         PARSER.declareObject(constructorArg(), (p, c) -> ElasticsearchException.fromXContent(p), new ParseField(REASON));
     }
 
-    private String index;
-
-    private int shardId;
-
-    private Throwable reason;
-
-    private RestStatus status;
-
     protected DefaultShardOperationFailedException() {
     }
 
     public DefaultShardOperationFailedException(ElasticsearchException e) {
-        Index index = e.getIndex();
-        this.index = index == null ? null : index.getName();
-        ShardId shardId = e.getShardId();
-        this.shardId = shardId == null ? -1 : shardId.id();
-        this.reason = e;
-        this.status = e.status();
+        super(e.getIndex() == null ? null : e.getIndex().getName(), e.getShardId() == null ? -1 : e.getShardId().getId(),
+            detailedMessage(e), e.status(), e);
     }
 
-    public DefaultShardOperationFailedException(String index, int shardId, Throwable reason) {
-        this.index = index;
-        this.shardId = shardId;
-        this.reason = reason;
-        this.status = ExceptionsHelper.status(reason);
-    }
-
-    @Override
-    public String index() {
-        return this.index;
-    }
-
-    @Override
-    public int shardId() {
-        return this.shardId;
-    }
-
-    @Override
-    public String reason() {
-        return detailedMessage(reason);
-    }
-
-    @Override
-    public RestStatus status() {
-        return status;
-    }
-
-    @Override
-    public Throwable getCause() {
-        return reason;
+    public DefaultShardOperationFailedException(String index, int shardId, Throwable cause) {
+        super(index, shardId, detailedMessage(cause), ExceptionsHelper.status(cause), cause);
     }
 
     public static DefaultShardOperationFailedException readShardOperationFailed(StreamInput in) throws IOException {
@@ -112,24 +70,17 @@ public static DefaultShardOperationFailedException readShardOperationFailed(Stre
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        if (in.readBoolean()) {
-            index = in.readString();
-        }
+        index = in.readOptionalString();
         shardId = in.readVInt();
-        reason = in.readException();
+        cause = in.readException();
         status = RestStatus.readFrom(in);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (index == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            out.writeString(index);
-        }
+        out.writeOptionalString(index);
         out.writeVInt(shardId);
-        out.writeException(reason);
+        out.writeException(cause);
         RestStatus.writeTo(out, status);
     }
 
@@ -145,7 +96,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field("status", status.name());
         if (reason != null) {
             builder.startObject("reason");
-            ElasticsearchException.generateThrowableXContent(builder, params, reason);
+            ElasticsearchException.generateThrowableXContent(builder, params, cause);
             builder.endObject();
         }
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
index 594dcda8c662f..654b398e79cd4 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
@@ -34,10 +34,9 @@
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 
 /**
- * Abstract class that allows to mark action responses that support acknowledgements.
- * Facilitates consistency across different api.
+ * A response that indicates that a request has been acknowledged
  */
-public abstract class AcknowledgedResponse extends ActionResponse implements ToXContentObject {
+public class AcknowledgedResponse extends ActionResponse implements ToXContentObject {
 
     private static final ParseField ACKNOWLEDGED = new ParseField("acknowledged");
 
@@ -48,11 +47,10 @@ protected static <T extends AcknowledgedResponse> void declareAcknowledgedField(
 
     protected boolean acknowledged;
 
-    protected AcknowledgedResponse() {
-
+    public AcknowledgedResponse() {
     }
 
-    protected AcknowledgedResponse(boolean acknowledged) {
+    public AcknowledgedResponse(boolean acknowledged) {
         this.acknowledged = acknowledged;
     }
 
@@ -100,10 +98,15 @@ protected void addCustomFields(XContentBuilder builder, Params params) throws IO
                 ObjectParser.ValueType.BOOLEAN);
     }
 
-    protected static boolean parseAcknowledged(XContentParser parser) {
+    @Deprecated
+    public static boolean parseAcknowledged(XContentParser parser) {
         return ACKNOWLEDGED_FLAG_PARSER.apply(parser, null);
     }
 
+    public static AcknowledgedResponse fromXContent(XContentParser parser) throws IOException {
+        return new AcknowledgedResponse(ACKNOWLEDGED_FLAG_PARSER.apply(parser, null));
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
index b8a5f3782bd64..bc5c696894a6a 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
@@ -218,13 +218,13 @@ public String toString() {
                 '}';
         }
 
-        public static ShardInfo readShardInfo(StreamInput in) throws IOException {
+        static ShardInfo readShardInfo(StreamInput in) throws IOException {
             ShardInfo shardInfo = new ShardInfo();
             shardInfo.readFrom(in);
             return shardInfo;
         }
 
-        public static class Failure implements ShardOperationFailedException, ToXContentObject {
+        public static class Failure extends ShardOperationFailedException implements ToXContentObject {
 
             private static final String _INDEX = "_index";
             private static final String _SHARD = "_shard";
@@ -235,37 +235,18 @@ public static class Failure implements ShardOperationFailedException, ToXContent
 
             private ShardId shardId;
             private String nodeId;
-            private Exception cause;
-            private RestStatus status;
             private boolean primary;
 
             public Failure(ShardId  shardId, @Nullable String nodeId, Exception cause, RestStatus status, boolean primary) {
+                super(shardId.getIndexName(), shardId.getId(), ExceptionsHelper.detailedMessage(cause), status, cause);
                 this.shardId = shardId;
                 this.nodeId = nodeId;
-                this.cause = cause;
-                this.status = status;
                 this.primary = primary;
             }
 
             Failure() {
             }
 
-            /**
-             * @return On what index the failure occurred.
-             */
-            @Override
-            public String index() {
-                return shardId.getIndexName();
-            }
-
-            /**
-             * @return On what shard id the failure occurred.
-             */
-            @Override
-            public int shardId() {
-                return shardId.id();
-            }
-
             public ShardId fullShardId() {
                 return shardId;
             }
@@ -278,27 +259,6 @@ public String nodeId() {
                 return nodeId;
             }
 
-            /**
-             * @return A text description of the failure
-             */
-            @Override
-            public String reason() {
-                return ExceptionsHelper.detailedMessage(cause);
-            }
-
-            /**
-             * @return The status to report if this failure was a primary failure.
-             */
-            @Override
-            public RestStatus status() {
-                return status;
-            }
-
-            @Override
-            public Throwable getCause() {
-                return cause;
-            }
-
             /**
              * @return Whether this failure occurred on a primary shard.
              * (this only reports true for delete by query)
@@ -310,6 +270,8 @@ public boolean primary() {
             @Override
             public void readFrom(StreamInput in) throws IOException {
                 shardId = ShardId.readShardId(in);
+                super.shardId = shardId.getId();
+                super.index = shardId.getIndexName();
                 nodeId = in.readOptionalString();
                 cause = in.readException();
                 status = RestStatus.readFrom(in);
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index 53d9752f4edc6..dbdd5acae1fc6 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -929,7 +929,7 @@ private void acquirePrimaryShardReference(ShardId shardId, String allocationId,
         if (actualAllocationId.equals(allocationId) == false) {
             throw new ShardNotFoundException(shardId, "expected aID [{}] but found [{}]", allocationId, actualAllocationId);
         }
-        final long actualTerm = indexShard.getPrimaryTerm();
+        final long actualTerm = indexShard.getPendingPrimaryTerm();
         if (actualTerm != primaryTerm) {
             throw new ShardNotFoundException(shardId, "expected aID [{}] with term [{}] but found [{}]", allocationId,
                 primaryTerm, actualTerm);
@@ -983,7 +983,7 @@ class PrimaryShardReference extends ShardReference
         }
 
         public boolean isRelocated() {
-            return indexShard.isPrimaryMode() == false;
+            return indexShard.isRelocatedPrimary();
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
index ca91a32a17a3a..ae029ce3f9357 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
@@ -83,7 +83,7 @@ protected static Location syncOperationResultOrThrow(final Engine.Result operati
         return location;
     }
 
-    protected static Location locationToSync(Location current, Location next) {
+    public static Location locationToSync(Location current, Location next) {
         /* here we are moving forward in the translog with each operation. Under the hood this might
          * cross translog files which is ok since from the user perspective the translog is like a
          * tape where only the highest location needs to be fsynced in order to sync all previous
diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
index 5212b1f35214c..77485f81e58c0 100644
--- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
+++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.action.update;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.LongSupplier;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.DocWriteResponse;
@@ -42,21 +47,22 @@
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.UpdateScript;
 import org.elasticsearch.search.lookup.SourceLookup;
 
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.function.LongSupplier;
+import static org.elasticsearch.common.Booleans.parseBoolean;
 
 /**
  * Helper for translating an update request to an index, delete request or update response.
  */
 public class UpdateHelper extends AbstractComponent {
+
+    /** Whether scripts should add the ctx variable to the params map. */
+    private static final boolean CTX_IN_PARAMS =
+        parseBoolean(System.getProperty("es.scripting.update.ctx_in_params"), true);
+
     private final ScriptService scriptService;
 
     public UpdateHelper(Settings settings, ScriptService scriptService) {
@@ -279,10 +285,18 @@ Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetRes
     private Map<String, Object> executeScript(Script script, Map<String, Object> ctx) {
         try {
             if (scriptService != null) {
-                ExecutableScript.Factory factory = scriptService.compile(script, ExecutableScript.UPDATE_CONTEXT);
-                ExecutableScript executableScript = factory.newInstance(script.getParams());
-                executableScript.setNextVar(ContextFields.CTX, ctx);
-                executableScript.run();
+                UpdateScript.Factory factory = scriptService.compile(script, UpdateScript.CONTEXT);
+                final Map<String, Object> params;
+                if (CTX_IN_PARAMS) {
+                    params = new HashMap<>(script.getParams());
+                    params.put(ContextFields.CTX, ctx);
+                    deprecationLogger.deprecated("Using `ctx` via `params.ctx` is deprecated. " +
+                        "Use -Des.scripting.update.ctx_in_params=false to enforce non-deprecated usage.");
+                } else {
+                    params = script.getParams();
+                }
+                UpdateScript executableScript = factory.newInstance(params);
+                executableScript.execute(ctx);
             }
         } catch (Exception e) {
             throw new IllegalArgumentException("failed to execute script", e);
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 870c537e020c5..bc2fe747c030b 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.logging.NodeNamePatternConverter;
 import org.elasticsearch.common.network.IfConfig;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.SecureSettings;
@@ -287,6 +288,10 @@ static void init(
 
         final SecureSettings keystore = loadSecureSettings(initialEnv);
         final Environment environment = createEnvironment(pidFile, keystore, initialEnv.settings(), initialEnv.configFile());
+
+        String nodeName = Node.NODE_NAME_SETTING.get(environment.settings());
+        NodeNamePatternConverter.setNodeName(nodeName);
+
         try {
             LogConfigurator.configure(environment);
         } catch (IOException e) {
@@ -317,8 +322,7 @@ static void init(
             // install the default uncaught exception handler; must be done before security is
             // initialized as we do not want to grant the runtime permission
             // setDefaultUncaughtExceptionHandler
-            Thread.setDefaultUncaughtExceptionHandler(
-                new ElasticsearchUncaughtExceptionHandler(() -> Node.NODE_NAME_SETTING.get(environment.settings())));
+            Thread.setDefaultUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler());
 
             INSTANCE.setup(true, environment);
 
@@ -341,10 +345,7 @@ static void init(
             if (foreground && maybeConsoleAppender != null) {
                 Loggers.removeAppender(rootLogger, maybeConsoleAppender);
             }
-            Logger logger = Loggers.getLogger(Bootstrap.class);
-            if (INSTANCE.node != null) {
-                logger = Loggers.getLogger(Bootstrap.class, Node.NODE_NAME_SETTING.get(INSTANCE.node.settings()));
-            }
+            Logger logger = LogManager.getLogger(Bootstrap.class);
             // HACK, it sucks to do this, but we will run users out of disk space otherwise
             if (e instanceof CreationException) {
                 // guice: log the shortened exc to the log file
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
index 79d6aca8fd195..2d576b949e43d 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.monitor.process.ProcessProbe;
-import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeValidationException;
 
 import java.io.BufferedReader;
@@ -76,8 +75,7 @@ static void check(final BootstrapContext context, final BoundTransportAddress bo
         combinedChecks.addAll(additionalChecks);
         check(  context,
                 enforceLimits(boundTransportAddress, DiscoveryModule.DISCOVERY_TYPE_SETTING.get(context.settings)),
-                Collections.unmodifiableList(combinedChecks),
-                Node.NODE_NAME_SETTING.get(context.settings));
+                Collections.unmodifiableList(combinedChecks));
     }
 
     /**
@@ -88,14 +86,12 @@ static void check(final BootstrapContext context, final BoundTransportAddress bo
      * @param context        the current node boostrap context
      * @param enforceLimits {@code true} if the checks should be enforced or otherwise warned
      * @param checks        the checks to execute
-     * @param nodeName      the node name to be used as a logging prefix
      */
     static void check(
         final BootstrapContext context,
         final boolean enforceLimits,
-        final List<BootstrapCheck> checks,
-        final String nodeName) throws NodeValidationException {
-        check(context, enforceLimits, checks, Loggers.getLogger(BootstrapChecks.class, nodeName));
+        final List<BootstrapCheck> checks) throws NodeValidationException {
+        check(context, enforceLimits, checks, Loggers.getLogger(BootstrapChecks.class));
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java b/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
index 857ff65b6c2b8..1ef9b7740c205 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
@@ -27,16 +27,9 @@
 import java.io.IOError;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
-import java.util.Objects;
-import java.util.function.Supplier;
 
 class ElasticsearchUncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
-
-    private final Supplier<String> loggingPrefixSupplier;
-
-    ElasticsearchUncaughtExceptionHandler(final Supplier<String> loggingPrefixSupplier) {
-        this.loggingPrefixSupplier = Objects.requireNonNull(loggingPrefixSupplier);
-    }
+    private static final Logger logger = Loggers.getLogger(ElasticsearchUncaughtExceptionHandler.class);
 
     @Override
     public void uncaughtException(Thread t, Throwable e) {
@@ -70,12 +63,10 @@ static boolean isFatalUncaught(Throwable e) {
     }
 
     void onFatalUncaught(final String threadName, final Throwable t) {
-        final Logger logger = Loggers.getLogger(ElasticsearchUncaughtExceptionHandler.class, loggingPrefixSupplier.get());
         logger.error(() -> new ParameterizedMessage("fatal error in thread [{}], exiting", threadName), t);
     }
 
     void onNonFatalUncaught(final String threadName, final Throwable t) {
-        final Logger logger = Loggers.getLogger(ElasticsearchUncaughtExceptionHandler.class, loggingPrefixSupplier.get());
         logger.warn(() -> new ParameterizedMessage("uncaught exception in thread [{}]", threadName), t);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
index 949b0110fff20..f6c71c90f9d14 100644
--- a/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
+++ b/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
@@ -113,7 +113,7 @@
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineRequestBuilder;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.tasks.TaskId;
@@ -574,12 +574,12 @@ public interface ClusterAdminClient extends ElasticsearchClient {
     /**
      * Stores an ingest pipeline
      */
-    void putPipeline(PutPipelineRequest request, ActionListener<WritePipelineResponse> listener);
+    void putPipeline(PutPipelineRequest request, ActionListener<AcknowledgedResponse> listener);
 
     /**
      * Stores an ingest pipeline
      */
-    ActionFuture<WritePipelineResponse> putPipeline(PutPipelineRequest request);
+    ActionFuture<AcknowledgedResponse> putPipeline(PutPipelineRequest request);
 
     /**
      * Stores an ingest pipeline
@@ -596,12 +596,12 @@ public interface ClusterAdminClient extends ElasticsearchClient {
     /**
      * Deletes a stored ingest pipeline
      */
-    void deletePipeline(DeletePipelineRequest request, ActionListener<WritePipelineResponse> listener);
+    void deletePipeline(DeletePipelineRequest request, ActionListener<AcknowledgedResponse> listener);
 
     /**
      * Deletes a stored ingest pipeline
      */
-    ActionFuture<WritePipelineResponse> deletePipeline(DeletePipelineRequest request);
+    ActionFuture<AcknowledgedResponse> deletePipeline(DeletePipelineRequest request);
 
     /**
      * Deletes a stored ingest pipeline
diff --git a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index 31e5e3190079d..3d508a3d9ba82 100644
--- a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -307,7 +307,6 @@
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineRequestBuilder;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
 import org.elasticsearch.action.search.ClearScrollAction;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClearScrollRequestBuilder;
@@ -325,6 +324,7 @@
 import org.elasticsearch.action.search.SearchScrollRequestBuilder;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.ThreadedActionListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsAction;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequestBuilder;
@@ -1082,12 +1082,12 @@ public SnapshotsStatusRequestBuilder prepareSnapshotStatus() {
         }
 
         @Override
-        public void putPipeline(PutPipelineRequest request, ActionListener<WritePipelineResponse> listener) {
+        public void putPipeline(PutPipelineRequest request, ActionListener<AcknowledgedResponse> listener) {
             execute(PutPipelineAction.INSTANCE, request, listener);
         }
 
         @Override
-        public ActionFuture<WritePipelineResponse> putPipeline(PutPipelineRequest request) {
+        public ActionFuture<AcknowledgedResponse> putPipeline(PutPipelineRequest request) {
             return execute(PutPipelineAction.INSTANCE, request);
         }
 
@@ -1102,12 +1102,12 @@ public PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference so
         }
 
         @Override
-        public void deletePipeline(DeletePipelineRequest request, ActionListener<WritePipelineResponse> listener) {
+        public void deletePipeline(DeletePipelineRequest request, ActionListener<AcknowledgedResponse> listener) {
             execute(DeletePipelineAction.INSTANCE, request, listener);
         }
 
         @Override
-        public ActionFuture<WritePipelineResponse> deletePipeline(DeletePipelineRequest request) {
+        public ActionFuture<AcknowledgedResponse> deletePipeline(DeletePipelineRequest request) {
             return execute(DeletePipelineAction.INSTANCE, request);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 9c5c642df6b91..8362198a12c4b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -116,23 +116,6 @@ public ClusterModule(Settings settings, ClusterService clusterService, List<Clus
         this.allocationService = new AllocationService(settings, allocationDeciders, shardsAllocator, clusterInfoService);
     }
 
-    public static Map<String, Supplier<ClusterState.Custom>> getClusterStateCustomSuppliers(List<ClusterPlugin> clusterPlugins) {
-        final Map<String, Supplier<ClusterState.Custom>> customSupplier = new HashMap<>();
-        customSupplier.put(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress::new);
-        customSupplier.put(RestoreInProgress.TYPE, RestoreInProgress::new);
-        customSupplier.put(SnapshotsInProgress.TYPE, SnapshotsInProgress::new);
-        for (ClusterPlugin plugin : clusterPlugins) {
-            Map<String, Supplier<ClusterState.Custom>> initialCustomSupplier = plugin.getInitialClusterStateCustomSupplier();
-            for (String key : initialCustomSupplier.keySet()) {
-                if (customSupplier.containsKey(key)) {
-                    throw new IllegalStateException("custom supplier key [" + key + "] is registered more than once");
-                }
-            }
-            customSupplier.putAll(initialCustomSupplier);
-        }
-        return Collections.unmodifiableMap(customSupplier);
-    }
-
     public static List<Entry> getNamedWriteables() {
         List<Entry> entries = new ArrayList<>();
         // Cluster State
diff --git a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index 998cd5ba0a870..2a9d960f8ccea 100644
--- a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -101,7 +101,7 @@ public void onFailure(Exception e) {
                     }
 
                     @Override
-                    protected void doRun() throws Exception {
+                    protected void doRun() {
                         try (Releasable ignored = nodeLocks.acquire(node)) {
                             validateAndConnectIfNeeded(node);
                         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
index 3bb9d42a5786d..8d0ad8efb7f5b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
@@ -26,9 +26,10 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.CompoundDateTimeFormatter;
+import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.xcontent.ContextParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
@@ -37,6 +38,8 @@
 import org.elasticsearch.index.Index;
 
 import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -365,6 +368,9 @@ public static final class Tombstone implements ToXContentObject, Writeable {
             TOMBSTONE_PARSER.declareString((b, s) -> {}, new ParseField(DELETE_DATE_KEY));
         }
 
+        static final CompoundDateTimeFormatter FORMATTER =
+            DateFormatters.forPattern("strict_date_optional_time").withZone(ZoneOffset.UTC);
+
         static ContextParser<Void, Tombstone> getParser() {
             return (parser, context) -> TOMBSTONE_PARSER.apply(parser, null).build();
         }
@@ -428,7 +434,8 @@ public int hashCode() {
 
         @Override
         public String toString() {
-            return "[index=" + index + ", deleteDate=" + Joda.getStrictStandardDateFormatter().printer().print(deleteDateInMillis) + "]";
+            String date = FORMATTER.format(Instant.ofEpochMilli(deleteDateInMillis));
+            return "[index=" + index + ", deleteDate=" + date + "]";
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
index a543f4c3d3b3e..ad715500a9ecd 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
@@ -28,17 +28,18 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.joda.FormatDateTimeFormatter;
-import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.CompoundDateTimeFormatter;
+import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneOffset;
 import java.util.Locale;
 import java.util.Objects;
 
@@ -47,7 +48,8 @@
  */
 public final class UnassignedInfo implements ToXContentFragment, Writeable {
 
-    public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime");
+    public static final CompoundDateTimeFormatter DATE_TIME_FORMATTER =
+        DateFormatters.forPattern("dateOptionalTime").withZone(ZoneOffset.UTC);
 
     public static final Setting<TimeValue> INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING =
         Setting.positiveTimeSetting("index.unassigned.node_left.delayed_timeout", TimeValue.timeValueMinutes(1), Property.Dynamic,
@@ -409,7 +411,7 @@ public static long findNextDelayedAllocation(long currentNanoTime, ClusterState
     public String shortSummary() {
         StringBuilder sb = new StringBuilder();
         sb.append("[reason=").append(reason).append("]");
-        sb.append(", at[").append(DATE_TIME_FORMATTER.printer().print(unassignedTimeMillis)).append("]");
+        sb.append(", at[").append(DATE_TIME_FORMATTER.format(Instant.ofEpochMilli(unassignedTimeMillis))).append("]");
         if (failedAllocations >  0) {
             sb.append(", failed_attempts[").append(failedAllocations).append("]");
         }
@@ -432,7 +434,7 @@ public String toString() {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject("unassigned_info");
         builder.field("reason", reason);
-        builder.field("at", DATE_TIME_FORMATTER.printer().print(unassignedTimeMillis));
+        builder.field("at", DATE_TIME_FORMATTER.format(Instant.ofEpochMilli(unassignedTimeMillis)));
         if (failedAllocations >  0) {
             builder.field("failed_attempts", failedAllocations);
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplier.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplier.java
index c587ab272e903..781adb3742a90 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplier.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplier.java
@@ -39,11 +39,6 @@ public interface ClusterApplier {
      */
     void onNewClusterState(String source, Supplier<ClusterState> clusterStateSupplier, ClusterApplyListener listener);
 
-    /**
-     * Creates a new cluster state builder that is initialized with the cluster name and all initial cluster state customs.
-     */
-    ClusterState.Builder newClusterStateBuilder();
-
     /**
      * Listener for results of cluster state application
      */
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
index 2fb7c25671c88..5dd36b9b1bc20 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
@@ -96,17 +96,14 @@ public class ClusterApplierService extends AbstractLifecycleComponent implements
     private final AtomicReference<ClusterState> state; // last applied state
 
     private NodeConnectionsService nodeConnectionsService;
-    private Supplier<ClusterState.Builder> stateBuilderSupplier;
 
-    public ClusterApplierService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool, Supplier<ClusterState
-        .Builder> stateBuilderSupplier) {
+    public ClusterApplierService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
         super(settings);
         this.clusterSettings = clusterSettings;
         this.threadPool = threadPool;
         this.state = new AtomicReference<>();
         this.slowTaskLoggingThreshold = CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(settings);
         this.localNodeMasterListeners = new LocalNodeMasterListeners(threadPool);
-        this.stateBuilderSupplier = stateBuilderSupplier;
     }
 
     public void setSlowTaskLoggingThreshold(TimeValue slowTaskLoggingThreshold) {
@@ -652,8 +649,4 @@ protected long currentTimeInNanos() {
         return System.nanoTime();
     }
 
-    @Override
-    public ClusterState.Builder newClusterStateBuilder() {
-        return stateBuilderSupplier.get();
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index 7610d75f677db..fc5dc678bd000 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -42,7 +42,6 @@
 
 import java.util.Collections;
 import java.util.Map;
-import java.util.function.Supplier;
 
 public class ClusterService extends AbstractLifecycleComponent {
 
@@ -59,10 +58,8 @@ public class ClusterService extends AbstractLifecycleComponent {
     private final OperationRouting operationRouting;
 
     private final ClusterSettings clusterSettings;
-    private final Map<String, Supplier<ClusterState.Custom>> initialClusterStateCustoms;
 
-    public ClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool,
-                          Map<String, Supplier<ClusterState.Custom>> initialClusterStateCustoms) {
+    public ClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
         super(settings);
         this.masterService = new MasterService(settings, threadPool);
         this.operationRouting = new OperationRouting(settings, clusterSettings);
@@ -70,19 +67,7 @@ public ClusterService(Settings settings, ClusterSettings clusterSettings, Thread
         this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         this.clusterSettings.addSettingsUpdateConsumer(CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
             this::setSlowTaskLoggingThreshold);
-        this.initialClusterStateCustoms = initialClusterStateCustoms;
-        this.clusterApplierService = new ClusterApplierService(settings, clusterSettings, threadPool, this::newClusterStateBuilder);
-    }
-
-    /**
-     * Creates a new cluster state builder that is initialized with the cluster name and all initial cluster state customs.
-     */
-    public ClusterState.Builder newClusterStateBuilder() {
-        ClusterState.Builder builder = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings));
-        for (Map.Entry<String, Supplier<ClusterState.Custom>> entry : initialClusterStateCustoms.entrySet()) {
-            builder.putCustom(entry.getKey(), entry.getValue().get());
-        }
-        return builder;
+        this.clusterApplierService = new ClusterApplierService(settings, clusterSettings, threadPool);
     }
 
     private void setSlowTaskLoggingThreshold(TimeValue slowTaskLoggingThreshold) {
diff --git a/server/src/main/java/org/elasticsearch/common/Rounding.java b/server/src/main/java/org/elasticsearch/common/Rounding.java
new file mode 100644
index 0000000000000..593964f61e93f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/Rounding.java
@@ -0,0 +1,530 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.io.IOException;
+import java.time.DayOfWeek;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoField;
+import java.time.temporal.ChronoUnit;
+import java.time.temporal.IsoFields;
+import java.time.temporal.TemporalField;
+import java.time.zone.ZoneOffsetTransition;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A strategy for rounding date/time based values.
+ *
+ * There are two implementations for rounding.
+ * The first one requires a date time unit and rounds to the supplied date time unit (i.e. quarter of year, day of month)
+ * The second one allows you to specify an interval to round to
+ */
+public abstract class Rounding implements Writeable {
+
+    public static String format(long epochMillis) {
+        return Instant.ofEpochMilli(epochMillis) + "/" + epochMillis;
+    }
+
+    public enum DateTimeUnit {
+        WEEK_OF_WEEKYEAR(   (byte) 1, IsoFields.WEEK_OF_WEEK_BASED_YEAR),
+        YEAR_OF_CENTURY(    (byte) 2, ChronoField.YEAR_OF_ERA),
+        QUARTER_OF_YEAR(    (byte) 3, IsoFields.QUARTER_OF_YEAR),
+        MONTH_OF_YEAR(      (byte) 4, ChronoField.MONTH_OF_YEAR),
+        DAY_OF_MONTH(       (byte) 5, ChronoField.DAY_OF_MONTH),
+        HOUR_OF_DAY(        (byte) 6, ChronoField.HOUR_OF_DAY),
+        MINUTES_OF_HOUR(    (byte) 7, ChronoField.MINUTE_OF_HOUR),
+        SECOND_OF_MINUTE(   (byte) 8, ChronoField.SECOND_OF_MINUTE);
+
+        private final byte id;
+        private final TemporalField field;
+
+        DateTimeUnit(byte id, TemporalField field) {
+            this.id = id;
+            this.field = field;
+        }
+
+        public byte getId() {
+            return id;
+        }
+
+        public TemporalField getField() {
+            return field;
+        }
+
+        public static DateTimeUnit resolve(byte id) {
+            switch (id) {
+                case 1: return WEEK_OF_WEEKYEAR;
+                case 2: return YEAR_OF_CENTURY;
+                case 3: return QUARTER_OF_YEAR;
+                case 4: return MONTH_OF_YEAR;
+                case 5: return DAY_OF_MONTH;
+                case 6: return HOUR_OF_DAY;
+                case 7: return MINUTES_OF_HOUR;
+                case 8: return SECOND_OF_MINUTE;
+                default: throw new ElasticsearchException("Unknown date time unit id [" + id + "]");
+            }
+        }
+    }
+
+    public abstract void innerWriteTo(StreamOutput out) throws IOException;
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeByte(id());
+        innerWriteTo(out);
+    }
+
+    public abstract byte id();
+
+    /**
+     * Rounds the given value.
+     */
+    public abstract long round(long value);
+
+    /**
+     * Given the rounded value (which was potentially generated by {@link #round(long)}, returns the next rounding value. For example, with
+     * interval based rounding, if the interval is 3, {@code nextRoundValue(6) = 9 }.
+     *
+     * @param value The current rounding value
+     * @return The next rounding value
+     */
+    public abstract long nextRoundingValue(long value);
+
+    @Override
+    public abstract boolean equals(Object obj);
+
+    @Override
+    public abstract int hashCode();
+
+    public static Builder builder(DateTimeUnit unit) {
+        return new Builder(unit);
+    }
+
+    public static Builder builder(TimeValue interval) {
+        return new Builder(interval);
+    }
+
+    public static class Builder {
+
+        private final DateTimeUnit unit;
+        private final long interval;
+
+        private ZoneId timeZone = ZoneOffset.UTC;
+
+        public Builder(DateTimeUnit unit) {
+            this.unit = unit;
+            this.interval = -1;
+        }
+
+        public Builder(TimeValue interval) {
+            this.unit = null;
+            if (interval.millis() < 1)
+                throw new IllegalArgumentException("Zero or negative time interval not supported");
+            this.interval = interval.millis();
+        }
+
+        public Builder timeZone(ZoneId timeZone) {
+            if (timeZone == null) {
+                throw new IllegalArgumentException("Setting null as timezone is not supported");
+            }
+            this.timeZone = timeZone;
+            return this;
+        }
+
+        public Rounding build() {
+            Rounding timeZoneRounding;
+            if (unit != null) {
+                timeZoneRounding = new TimeUnitRounding(unit, timeZone);
+            } else {
+                timeZoneRounding = new TimeIntervalRounding(interval, timeZone);
+            }
+            return timeZoneRounding;
+        }
+    }
+
+    static class TimeUnitRounding extends Rounding {
+
+        static final byte ID = 1;
+
+        private final DateTimeUnit unit;
+        private final ZoneId timeZone;
+        private final boolean unitRoundsToMidnight;
+
+
+        TimeUnitRounding(DateTimeUnit unit, ZoneId timeZone) {
+            this.unit = unit;
+            this.timeZone = timeZone;
+            this.unitRoundsToMidnight = this.unit.field.getBaseUnit().getDuration().toMillis() > 60L * 60L * 1000L;
+        }
+
+        TimeUnitRounding(StreamInput in) throws IOException {
+            unit = DateTimeUnit.resolve(in.readByte());
+            timeZone = ZoneId.of(in.readString());
+            unitRoundsToMidnight = unit.getField().getBaseUnit().getDuration().toMillis() > 60L * 60L * 1000L;
+        }
+
+        @Override
+        public byte id() {
+            return ID;
+        }
+
+        private LocalDateTime truncateLocalDateTime(LocalDateTime localDateTime) {
+            localDateTime = localDateTime.withNano(0);
+            assert localDateTime.getNano() == 0;
+            if (unit.equals(DateTimeUnit.SECOND_OF_MINUTE)) {
+                return localDateTime;
+            }
+
+            localDateTime = localDateTime.withSecond(0);
+            assert localDateTime.getSecond() == 0;
+            if (unit.equals(DateTimeUnit.MINUTES_OF_HOUR)) {
+                return localDateTime;
+            }
+
+            localDateTime = localDateTime.withMinute(0);
+            assert localDateTime.getMinute() == 0;
+            if (unit.equals(DateTimeUnit.HOUR_OF_DAY)) {
+                return localDateTime;
+            }
+
+            localDateTime = localDateTime.withHour(0);
+            assert localDateTime.getHour() == 0;
+            if (unit.equals(DateTimeUnit.DAY_OF_MONTH)) {
+                return localDateTime;
+            }
+
+            if (unit.equals(DateTimeUnit.WEEK_OF_WEEKYEAR)) {
+                localDateTime = localDateTime.with(ChronoField.DAY_OF_WEEK, 1);
+                assert localDateTime.getDayOfWeek() == DayOfWeek.MONDAY;
+                return localDateTime;
+            }
+
+            localDateTime = localDateTime.withDayOfMonth(1);
+            assert localDateTime.getDayOfMonth() == 1;
+            if (unit.equals(DateTimeUnit.MONTH_OF_YEAR)) {
+                return localDateTime;
+            }
+
+            if (unit.equals(DateTimeUnit.QUARTER_OF_YEAR)) {
+                int quarter = (int) IsoFields.QUARTER_OF_YEAR.getFrom(localDateTime);
+                int month = ((quarter - 1) * 3) + 1;
+                localDateTime = localDateTime.withMonth(month);
+                assert localDateTime.getMonthValue() % 3 == 1;
+                return localDateTime;
+            }
+
+            if (unit.equals(DateTimeUnit.YEAR_OF_CENTURY)) {
+                localDateTime = localDateTime.withMonth(1);
+                assert localDateTime.getMonthValue() == 1;
+                return localDateTime;
+            }
+
+            throw new IllegalArgumentException("NOT YET IMPLEMENTED for unit " + unit);
+        }
+
+        @Override
+        public long round(long utcMillis) {
+            if (unitRoundsToMidnight) {
+                final ZonedDateTime zonedDateTime = Instant.ofEpochMilli(utcMillis).atZone(timeZone);
+                final LocalDateTime localDateTime = zonedDateTime.toLocalDateTime();
+                final LocalDateTime localMidnight = truncateLocalDateTime(localDateTime);
+                return firstTimeOnDay(localMidnight);
+            } else {
+                while (true) {
+                    final Instant truncatedTime = truncateAsLocalTime(utcMillis);
+                    final ZoneOffsetTransition previousTransition = timeZone.getRules().previousTransition(Instant.ofEpochMilli(utcMillis));
+
+                    if (previousTransition == null) {
+                        // truncateAsLocalTime cannot have failed if there were no previous transitions
+                        return truncatedTime.toEpochMilli();
+                    }
+
+                    final long previousTransitionMillis = previousTransition.getInstant().toEpochMilli();
+
+                    if (truncatedTime != null && previousTransitionMillis <= truncatedTime.toEpochMilli()) {
+                        return truncatedTime.toEpochMilli();
+                    }
+
+                    // There was a transition in between the input time and the truncated time. Return to the transition time and
+                    // round that down instead.
+                    utcMillis = previousTransitionMillis - 1;
+                }
+            }
+        }
+
+        private long firstTimeOnDay(LocalDateTime localMidnight) {
+            assert localMidnight.toLocalTime().equals(LocalTime.of(0, 0, 0)) : "firstTimeOnDay should only be called at midnight";
+            assert unitRoundsToMidnight : "firstTimeOnDay should only be called if unitRoundsToMidnight";
+
+            // Now work out what localMidnight actually means
+            final List<ZoneOffset> currentOffsets = timeZone.getRules().getValidOffsets(localMidnight);
+            if (currentOffsets.size() >= 1) {
+                // There is at least one midnight on this day, so choose the first
+                final ZoneOffset firstOffset = currentOffsets.get(0);
+                final OffsetDateTime offsetMidnight = localMidnight.atOffset(firstOffset);
+                return offsetMidnight.toInstant().toEpochMilli();
+            } else {
+                // There were no midnights on this day, so we must have entered the day via an offset transition.
+                // Use the time of the transition as it is the earliest time on the right day.
+                ZoneOffsetTransition zoneOffsetTransition = timeZone.getRules().getTransition(localMidnight);
+                return zoneOffsetTransition.getInstant().toEpochMilli();
+            }
+        }
+
+        private Instant truncateAsLocalTime(long utcMillis) {
+            assert unitRoundsToMidnight == false : "truncateAsLocalTime should not be called if unitRoundsToMidnight";
+
+            final LocalDateTime truncatedLocalDateTime
+                = truncateLocalDateTime(Instant.ofEpochMilli(utcMillis).atZone(timeZone).toLocalDateTime());
+            final List<ZoneOffset> currentOffsets = timeZone.getRules().getValidOffsets(truncatedLocalDateTime);
+
+            if (currentOffsets.size() >= 1) {
+                // at least one possibilities - choose the latest one that's still no later than the input time
+                for (int offsetIndex = currentOffsets.size() - 1; offsetIndex >= 0; offsetIndex--) {
+                    final Instant result = truncatedLocalDateTime.atOffset(currentOffsets.get(offsetIndex)).toInstant();
+                    if (result.toEpochMilli() <= utcMillis) {
+                        return result;
+                    }
+                }
+
+                assert false : "rounded time not found for " + utcMillis + " with " + this;
+                return null;
+            } else {
+                // The chosen local time didn't happen. This means we were given a time in an hour (or a minute) whose start
+                // is missing due to an offset transition, so the time cannot be truncated.
+                return null;
+            }
+        }
+
+        private LocalDateTime nextRelevantMidnight(LocalDateTime localMidnight) {
+            assert localMidnight.toLocalTime().equals(LocalTime.of(0, 0, 0)) : "nextRelevantMidnight should only be called at midnight";
+            assert unitRoundsToMidnight : "firstTimeOnDay should only be called if unitRoundsToMidnight";
+
+            switch (unit) {
+                case DAY_OF_MONTH:
+                    return localMidnight.plus(1, ChronoUnit.DAYS);
+                case WEEK_OF_WEEKYEAR:
+                    return localMidnight.plus(7, ChronoUnit.DAYS);
+                case MONTH_OF_YEAR:
+                    return localMidnight.plus(1, ChronoUnit.MONTHS);
+                case QUARTER_OF_YEAR:
+                    return localMidnight.plus(3, ChronoUnit.MONTHS);
+                case YEAR_OF_CENTURY:
+                    return localMidnight.plus(1, ChronoUnit.YEARS);
+                default:
+                    throw new IllegalArgumentException("Unknown round-to-midnight unit: " + unit);
+            }
+        }
+
+        @Override
+        public long nextRoundingValue(long utcMillis) {
+            if (unitRoundsToMidnight) {
+                final ZonedDateTime zonedDateTime = Instant.ofEpochMilli(utcMillis).atZone(timeZone);
+                final LocalDateTime localDateTime = zonedDateTime.toLocalDateTime();
+                final LocalDateTime earlierLocalMidnight = truncateLocalDateTime(localDateTime);
+                final LocalDateTime localMidnight = nextRelevantMidnight(earlierLocalMidnight);
+                return firstTimeOnDay(localMidnight);
+            } else {
+                final long unitSize = unit.field.getBaseUnit().getDuration().toMillis();
+                final long roundedAfterOneIncrement = round(utcMillis + unitSize);
+                if (utcMillis < roundedAfterOneIncrement) {
+                    return roundedAfterOneIncrement;
+                } else {
+                    return round(utcMillis + 2 * unitSize);
+                }
+            }
+        }
+
+        @Override
+        public void innerWriteTo(StreamOutput out) throws IOException {
+            out.writeByte(unit.getId());
+            String tz = ZoneOffset.UTC.equals(timeZone) ? "UTC" : timeZone.getId(); // stay joda compatible
+            out.writeString(tz);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(unit, timeZone);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            TimeUnitRounding other = (TimeUnitRounding) obj;
+            return Objects.equals(unit, other.unit) && Objects.equals(timeZone, other.timeZone);
+        }
+
+        @Override
+        public String toString() {
+            return "[" + timeZone + "][" + unit + "]";
+        }
+    }
+
+    static class TimeIntervalRounding extends Rounding {
+        @Override
+        public String toString() {
+            return "TimeIntervalRounding{" +
+                "interval=" + interval +
+                ", timeZone=" + timeZone +
+                '}';
+        }
+
+        static final byte ID = 2;
+
+        private final long interval;
+        private final ZoneId timeZone;
+
+        TimeIntervalRounding(long interval, ZoneId timeZone) {
+            if (interval < 1)
+                throw new IllegalArgumentException("Zero or negative time interval not supported");
+            this.interval = interval;
+            this.timeZone = timeZone;
+        }
+
+        TimeIntervalRounding(StreamInput in) throws IOException {
+            interval = in.readVLong();
+            timeZone = ZoneId.of(in.readString());
+        }
+
+        @Override
+        public byte id() {
+            return ID;
+        }
+
+        @Override
+        public long round(final long utcMillis) {
+            final Instant utcInstant = Instant.ofEpochMilli(utcMillis);
+            final LocalDateTime rawLocalDateTime = Instant.ofEpochMilli(utcMillis).atZone(timeZone).toLocalDateTime();
+
+            // a millisecond value with the same local time, in UTC, as `utcMillis` has in `timeZone`
+            final long localMillis = utcMillis + timeZone.getRules().getOffset(utcInstant).getTotalSeconds() * 1000;
+            assert localMillis == rawLocalDateTime.toInstant(ZoneOffset.UTC).toEpochMilli();
+
+            final long roundedMillis = roundKey(localMillis, interval) * interval;
+            final LocalDateTime roundedLocalDateTime = Instant.ofEpochMilli(roundedMillis).atZone(ZoneOffset.UTC).toLocalDateTime();
+
+            // Now work out what roundedLocalDateTime actually means
+            final List<ZoneOffset> currentOffsets = timeZone.getRules().getValidOffsets(roundedLocalDateTime);
+            if (currentOffsets.isEmpty() == false) {
+                // There is at least one instant with the desired local time. In general the desired result is
+                // the latest rounded time that's no later than the input time, but this could involve rounding across
+                // a timezone transition, which may yield the wrong result
+                final ZoneOffsetTransition previousTransition = timeZone.getRules().previousTransition(utcInstant.plusMillis(1));
+                for (int offsetIndex = currentOffsets.size() - 1; 0 <= offsetIndex; offsetIndex--) {
+                    final OffsetDateTime offsetTime = roundedLocalDateTime.atOffset(currentOffsets.get(offsetIndex));
+                    final Instant offsetInstant = offsetTime.toInstant();
+                    if (previousTransition != null && offsetInstant.isBefore(previousTransition.getInstant())) {
+                        // Rounding down across the transition can yield the wrong result. It's best to return to the transition time
+                        // and round that down.
+                        return round(previousTransition.getInstant().toEpochMilli() - 1);
+                    }
+
+                    if (utcInstant.isBefore(offsetTime.toInstant()) == false) {
+                        return offsetInstant.toEpochMilli();
+                    }
+                }
+
+                final OffsetDateTime offsetTime = roundedLocalDateTime.atOffset(currentOffsets.get(0));
+                final Instant offsetInstant = offsetTime.toInstant();
+                assert false : this + " failed to round " + utcMillis + " down: " + offsetInstant + " is the earliest possible";
+                return offsetInstant.toEpochMilli(); // TODO or throw something?
+            } else {
+                // The desired time isn't valid because within a gap, so just return the gap time.
+                ZoneOffsetTransition zoneOffsetTransition = timeZone.getRules().getTransition(roundedLocalDateTime);
+                return zoneOffsetTransition.getInstant().toEpochMilli();
+            }
+        }
+
+        private static long roundKey(long value, long interval) {
+            if (value < 0) {
+                return (value - interval + 1) / interval;
+            } else {
+                return value / interval;
+            }
+        }
+
+        @Override
+        public long nextRoundingValue(long time) {
+            int offsetSeconds = timeZone.getRules().getOffset(Instant.ofEpochMilli(time)).getTotalSeconds();
+            return ZonedDateTime.ofInstant(Instant.ofEpochMilli(time), ZoneOffset.UTC)
+                .plusSeconds(offsetSeconds)
+                .plusNanos(interval * 1_000_000)
+                .withZoneSameLocal(timeZone)
+                .toInstant().toEpochMilli();
+        }
+
+        @Override
+        public void innerWriteTo(StreamOutput out) throws IOException {
+            out.writeVLong(interval);
+            String tz = ZoneOffset.UTC.equals(timeZone) ? "UTC" : timeZone.getId(); // stay joda compatible
+            out.writeString(tz);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(interval, timeZone);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            TimeIntervalRounding other = (TimeIntervalRounding) obj;
+            return Objects.equals(interval, other.interval) && Objects.equals(timeZone, other.timeZone);
+        }
+    }
+
+    public static Rounding read(StreamInput in) throws IOException {
+        Rounding rounding;
+        byte id = in.readByte();
+        switch (id) {
+            case TimeUnitRounding.ID:
+                rounding = new TimeUnitRounding(in);
+                break;
+            case TimeIntervalRounding.ID:
+                rounding = new TimeIntervalRounding(in);
+                break;
+            default:
+                throw new ElasticsearchException("unknown rounding id [" + id + "]");
+        }
+        return rounding;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/Table.java b/server/src/main/java/org/elasticsearch/common/Table.java
index 430070ee19cca..13d13066e16d6 100644
--- a/server/src/main/java/org/elasticsearch/common/Table.java
+++ b/server/src/main/java/org/elasticsearch/common/Table.java
@@ -19,9 +19,11 @@
 
 package org.elasticsearch.common;
 
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
+import org.elasticsearch.common.time.CompoundDateTimeFormatter;
+import org.elasticsearch.common.time.DateFormatters;
 
+import java.time.Instant;
+import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -83,7 +85,7 @@ public Table endHeaders() {
         return this;
     }
 
-    private DateTimeFormatter dateFormat = DateTimeFormat.forPattern("HH:mm:ss");
+    private static final CompoundDateTimeFormatter FORMATTER = DateFormatters.forPattern("HH:mm:ss").withZone(ZoneOffset.UTC);
 
     public Table startRow() {
         if (headers.isEmpty()) {
@@ -93,7 +95,7 @@ public Table startRow() {
         if (withTime) {
             long time = System.currentTimeMillis();
             addCell(TimeUnit.SECONDS.convert(time, TimeUnit.MILLISECONDS));
-            addCell(dateFormat.print(time));
+            addCell(FORMATTER.format(Instant.ofEpochMilli(time)));
         }
         return this;
     }
diff --git a/server/src/main/java/org/elasticsearch/common/component/AbstractComponent.java b/server/src/main/java/org/elasticsearch/common/component/AbstractComponent.java
index 62d6e7e311d5d..167ffdb7bea25 100644
--- a/server/src/main/java/org/elasticsearch/common/component/AbstractComponent.java
+++ b/server/src/main/java/org/elasticsearch/common/component/AbstractComponent.java
@@ -23,7 +23,6 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.node.Node;
 
@@ -34,13 +33,7 @@ public abstract class AbstractComponent {
     protected final Settings settings;
 
     public AbstractComponent(Settings settings) {
-        this.logger = Loggers.getLogger(getClass(), settings);
-        this.deprecationLogger = new DeprecationLogger(logger);
-        this.settings = settings;
-    }
-
-    public AbstractComponent(Settings settings, Class<?> customClass) {
-        this.logger = LogManager.getLogger(customClass);
+        this.logger = LogManager.getLogger(getClass());
         this.deprecationLogger = new DeprecationLogger(logger);
         this.settings = settings;
     }
diff --git a/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java b/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
index 3c4b35d5c3477..8a472954ab492 100644
--- a/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
+++ b/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
@@ -35,10 +35,6 @@ protected AbstractLifecycleComponent(Settings settings) {
         super(settings);
     }
 
-    protected AbstractLifecycleComponent(Settings settings, Class<?> customClass) {
-        super(settings, customClass);
-    }
-
     @Override
     public Lifecycle.State lifecycleState() {
         return this.lifecycle.state();
diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
index dbcb46d9936f3..e4d48b8df4566 100644
--- a/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
+++ b/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
@@ -46,6 +46,8 @@
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.apache.lucene.geo.GeoUtils.orient;
+
 /**
  * The {@link PolygonBuilder} implements the groundwork to create polygons. This contains
  * Methods to wrap polygons at the dateline and building shapes from the data held by the
@@ -642,14 +644,8 @@ private static int createEdges(int component, Orientation orientation, LineStrin
      */
     private static Edge[] ring(int component, boolean direction, boolean handedness,
                                  Coordinate[] points, int offset, Edge[] edges, int toffset, int length, final AtomicBoolean translated) {
-        // calculate the direction of the points:
-        // find the point a the top of the set and check its
-        // neighbors orientation. So direction is equivalent
-        // to clockwise/counterclockwise
-        final int top = top(points, offset, length);
-        final int prev = (offset + ((top + length - 1) % length));
-        final int next = (offset + ((top + 1) % length));
-        boolean orientation = points[offset + prev].x > points[offset + next].x;
+
+        boolean orientation = getOrientation(points, offset, length);
 
         // OGC requires shell as ccw (Right-Handedness) and holes as cw (Left-Handedness)
         // since GeoJSON doesn't specify (and doesn't need to) GEO core will assume OGC standards
@@ -678,6 +674,35 @@ private static Edge[] ring(int component, boolean direction, boolean handedness,
         return concat(component, direction ^ orientation, points, offset, edges, toffset, length);
     }
 
+    /**
+     * @return whether the points are clockwise (true) or anticlockwise (false)
+     */
+    private static boolean getOrientation(Coordinate[] points, int offset, int length) {
+        // calculate the direction of the points: find the southernmost point
+        // and check its neighbors orientation.
+
+        final int top = top(points, offset, length);
+        final int prev = (top + length - 1) % length;
+        final int next = (top + 1) % length;
+
+        final int determinantSign = orient(
+            points[offset + prev].x, points[offset + prev].y,
+            points[offset + top].x, points[offset + top].y,
+            points[offset + next].x, points[offset + next].y);
+
+        if (determinantSign == 0) {
+            // Points are collinear, but `top` is not in the middle if so, so the edges either side of `top` are intersecting.
+            throw new InvalidShapeException("Cannot determine orientation: edges adjacent to ("
+                + points[offset + top].x + "," + points[offset +top].y + ") coincide");
+        }
+
+        return determinantSign < 0;
+    }
+
+    /**
+     * @return the (offset) index of the point that is furthest west amongst
+     * those points that are the furthest south in the set.
+     */
     private static int top(Coordinate[] points, int offset, int length) {
         int top = 0; // we start at 1 here since top points to 0
         for (int i = 1; i < length; i++) {
diff --git a/server/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java b/server/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
index 44d7d17b59325..2159014f8258f 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
@@ -38,25 +38,53 @@ public static Logger getLogger(String prefix, String name) {
 
     public static Logger getLogger(String prefix, Class<?> clazz) {
         /*
-         * Do not use LogManager#getLogger(Class) as this now uses Class#getCanonicalName under the hood; as this returns null for local and
-         * anonymous classes, any place we create, for example, an abstract component defined as an anonymous class (e.g., in tests) will
-         * result in a logger with a null name which will blow up in a lookup inside of Log4j.
+         * At one point we didn't use LogManager.getLogger(clazz) because
+         * of a bug in log4j that has since been fixed:
+         * https://github.com/apache/logging-log4j2/commit/ae33698a1846a5e10684ec3e52a99223f06047af
+         *
+         * For now we continue to use LogManager.getLogger(clazz.getName())
+         * because we expect to eventually migrate away from needing this
+         * method entirely.
          */
         return getLogger(prefix, LogManager.getLogger(clazz.getName()));
     }
 
     public static Logger getLogger(String prefix, Logger logger) {
+        /*
+         * In a followup we'll throw an exception if prefix is null or empty
+         * redirecting folks to LogManager.getLogger.
+         *
+         * This and more is tracked in https://github.com/elastic/elasticsearch/issues/32174
+         */
+        if (prefix == null || prefix.length() == 0) {
+            return logger;
+        }
         return new PrefixLogger((ExtendedLogger)logger, logger.getName(), prefix);
     }
 
+    /**
+     * Get or build a logger.
+     * @deprecated Prefer {@link LogManager#getLogger}
+     */
+    @Deprecated
     public static Logger getLogger(Class<?> clazz) {
         return getLogger(null, clazz);
     }
 
+    /**
+     * Get or build a logger.
+     * @deprecated Prefer {@link LogManager#getLogger}
+     */
+    @Deprecated
     public static Logger getLogger(String name) {
         return getLogger(null, name);
     }
 
+    /**
+     * Get the root logger.
+     * @deprecated Prefer {@link LogManager#getRootLogger}
+     */
+    @Deprecated
     public static Logger getRootLogger() {
         return LogManager.getRootLogger();
     }
diff --git a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
index 01aca53db051d..4e3771f36680d 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
@@ -23,12 +23,16 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.apache.logging.log4j.core.config.AbstractConfiguration;
+import org.apache.logging.log4j.core.config.ConfigurationException;
+import org.apache.logging.log4j.core.config.ConfigurationSource;
 import org.apache.logging.log4j.core.config.Configurator;
 import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
 import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory;
 import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
 import org.apache.logging.log4j.core.config.composite.CompositeConfiguration;
+import org.apache.logging.log4j.core.config.plugins.util.PluginManager;
 import org.apache.logging.log4j.core.config.properties.PropertiesConfiguration;
+import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationBuilder;
 import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationFactory;
 import org.apache.logging.log4j.status.StatusConsoleListener;
 import org.apache.logging.log4j.status.StatusData;
@@ -43,6 +47,7 @@
 import org.elasticsearch.node.Node;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.FileVisitOption;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
@@ -50,9 +55,12 @@
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.EnumSet;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.StreamSupport;
@@ -119,6 +127,13 @@ public static void configure(final Environment environment) throws IOException,
         configure(environment.settings(), environment.configFile(), environment.logsFile());
     }
 
+    /**
+     * Load logging plugins so we can have {@code node_name} in the pattern.
+     */
+    public static void loadLog4jPlugins() {
+        PluginManager.addPackage(LogConfigurator.class.getPackage().getName());
+    }
+
     private static void checkErrorListener() {
         assert errorListenerIsRegistered() : "expected error listener to be registered";
         if (error.get()) {
@@ -135,6 +150,8 @@ private static void configure(final Settings settings, final Path configsPath, f
         Objects.requireNonNull(configsPath);
         Objects.requireNonNull(logsPath);
 
+        loadLog4jPlugins();
+
         setLogConfigurationSystemProperty(logsPath, settings);
         // we initialize the status logger immediately otherwise Log4j will complain when we try to get the context
         configureStatusLogger();
@@ -142,7 +159,53 @@ private static void configure(final Settings settings, final Path configsPath, f
         final LoggerContext context = (LoggerContext) LogManager.getContext(false);
 
         final List<AbstractConfiguration> configurations = new ArrayList<>();
-        final PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory();
+
+        /*
+         * Subclass the properties configurator to hack the new pattern in
+         * place so users don't have to change log4j2.properties in
+         * a minor release. In 7.0 we'll remove this and force users to
+         * change log4j2.properties. If they don't customize log4j2.properties
+         * then they won't have to do anything anyway.
+         *
+         * Everything in this subclass that isn't marked as a hack is copied
+         * from log4j2's source.
+         */
+        Set<String> locationsWithDeprecatedPatterns = Collections.synchronizedSet(new HashSet<>());
+        final PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory() {
+            @Override
+            public PropertiesConfiguration getConfiguration(final LoggerContext loggerContext, final ConfigurationSource source) {
+                final Properties properties = new Properties();
+                try (InputStream configStream = source.getInputStream()) {
+                    properties.load(configStream);
+                } catch (final IOException ioe) {
+                    throw new ConfigurationException("Unable to load " + source.toString(), ioe);
+                }
+                // Hack the new pattern into place
+                for (String name : properties.stringPropertyNames()) {
+                    if (false == name.endsWith(".pattern")) continue;
+                    // Null is weird here but we can't do anything with it so ignore it
+                    String value = properties.getProperty(name);
+                    if (value == null) continue;
+                    // Tests don't need to be changed
+                    if (value.contains("%test_thread_info")) continue;
+                    /*
+                     * Patterns without a marker are sufficiently customized
+                     * that we don't have an opinion about them.
+                     */
+                    if (false == value.contains("%marker")) continue;
+                    if (false == value.contains("%node_name")) {
+                        locationsWithDeprecatedPatterns.add(source.getLocation());
+                        properties.setProperty(name, value.replace("%marker", "[%node_name]%marker "));
+                    }
+                }
+                // end hack
+                return new PropertiesConfigurationBuilder()
+                        .setConfigurationSource(source)
+                        .setRootProperties(properties)
+                        .setLoggerContext(loggerContext)
+                        .build();
+            }
+        };
         final Set<FileVisitOption> options = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
         Files.walkFileTree(configsPath, options, Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
             @Override
@@ -163,6 +226,14 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr
         context.start(new CompositeConfiguration(configurations));
 
         configureLoggerLevels(settings);
+
+        final String deprecatedLocationsString = String.join("\n  ", locationsWithDeprecatedPatterns);
+        if (deprecatedLocationsString.length() > 0) {
+            LogManager.getLogger(LogConfigurator.class).warn("Some logging configurations have %marker but don't have %node_name. "
+                    + "We will automatically add %node_name to the pattern to ease the migration for users who customize "
+                    + "log4j2.properties but will stop this behavior in 7.0. You should manually replace `%node_name` with "
+                    + "`[%node_name]%marker ` in these locations:\n  {}", deprecatedLocationsString);
+        }
     }
 
     private static void configureStatusLogger() {
diff --git a/server/src/main/java/org/elasticsearch/common/logging/Loggers.java b/server/src/main/java/org/elasticsearch/common/logging/Loggers.java
index 40983c517c72f..b2a24faf643fe 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/Loggers.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/Loggers.java
@@ -31,13 +31,9 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.node.Node;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 
-import static java.util.Arrays.asList;
 import static org.elasticsearch.common.util.CollectionUtils.asArrayList;
 
 /**
@@ -71,47 +67,39 @@ public static Logger getLogger(Class<?> clazz, Settings settings, Index index, S
     }
 
     public static Logger getLogger(Class<?> clazz, Settings settings, String... prefixes) {
-        final List<String> prefixesList = prefixesList(settings, prefixes);
-        return Loggers.getLogger(clazz, prefixesList.toArray(new String[prefixesList.size()]));
+        return ESLoggerFactory.getLogger(formatPrefix(prefixes), clazz);
     }
 
     public static Logger getLogger(String loggerName, Settings settings, String... prefixes) {
-        final List<String> prefixesList = prefixesList(settings, prefixes);
-        return Loggers.getLogger(loggerName, prefixesList.toArray(new String[prefixesList.size()]));
-    }
-
-    private static List<String> prefixesList(Settings settings, String... prefixes) {
-        List<String> prefixesList = new ArrayList<>();
-        if (Node.NODE_NAME_SETTING.exists(settings)) {
-            prefixesList.add(Node.NODE_NAME_SETTING.get(settings));
-        }
-        if (prefixes != null && prefixes.length > 0) {
-            prefixesList.addAll(asList(prefixes));
-        }
-        return prefixesList;
+        return ESLoggerFactory.getLogger(formatPrefix(prefixes), loggerName);
     }
 
     public static Logger getLogger(Logger parentLogger, String s) {
-        assert parentLogger instanceof PrefixLogger;
-        return ESLoggerFactory.getLogger(((PrefixLogger)parentLogger).prefix(), parentLogger.getName() + s);
+        String prefix = null;
+        if (parentLogger instanceof PrefixLogger) {
+            prefix = ((PrefixLogger)parentLogger).prefix();
+        }
+        return ESLoggerFactory.getLogger(prefix, parentLogger.getName() + s);
     }
 
+    /**
+     * Get or build a logger.
+     * @deprecated Prefer {@link LogManager#getLogger}
+     */
+    @Deprecated
     public static Logger getLogger(String s) {
         return ESLoggerFactory.getLogger(s);
     }
 
+    /**
+     * Get or build a logger.
+     * @deprecated Prefer {@link LogManager#getLogger}
+     */
+    @Deprecated
     public static Logger getLogger(Class<?> clazz) {
         return ESLoggerFactory.getLogger(clazz);
     }
 
-    public static Logger getLogger(Class<?> clazz, String... prefixes) {
-        return ESLoggerFactory.getLogger(formatPrefix(prefixes), clazz);
-    }
-
-    public static Logger getLogger(String name, String... prefixes) {
-        return ESLoggerFactory.getLogger(formatPrefix(prefixes), name);
-    }
-
     private static String formatPrefix(String... prefixes) {
         String prefix = null;
         if (prefixes != null && prefixes.length > 0) {
@@ -126,7 +114,6 @@ private static String formatPrefix(String... prefixes) {
                 }
             }
             if (sb.length() > 0) {
-                sb.append(" ");
                 prefix = sb.toString();
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/common/logging/NodeNamePatternConverter.java b/server/src/main/java/org/elasticsearch/common/logging/NodeNamePatternConverter.java
new file mode 100644
index 0000000000000..ca4c9ab776f6e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/logging/NodeNamePatternConverter.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.logging;
+
+import java.util.Arrays;
+
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.pattern.ConverterKeys;
+import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
+import org.apache.logging.log4j.core.pattern.PatternConverter;
+import org.apache.lucene.util.SetOnce;
+
+/**
+ * Converts {@code %node_name} in log4j patterns into the current node name.
+ * We *could* use a system property lookup instead but this is very explicit
+ * and fails fast if we try to use the logger without initializing the node
+ * name. As a bonus it ought to be ever so slightly faster because it doesn't
+ * have to look up the system property every time.
+ */
+@Plugin(category = PatternConverter.CATEGORY, name = "NodeNamePatternConverter")
+@ConverterKeys({"node_name"})
+public class NodeNamePatternConverter extends LogEventPatternConverter {
+    private static final SetOnce<String> NODE_NAME = new SetOnce<>();
+
+    /**
+     * Set the name of this node.
+     */
+    public static void setNodeName(String nodeName) {
+        NODE_NAME.set(nodeName);
+    }
+
+    /**
+     * Called by log4j2 to initialize this converter.
+     */
+    public static NodeNamePatternConverter newInstance(final String[] options) {
+        if (options.length > 0) {
+            throw new IllegalArgumentException("no options supported but options provided: "
+                    + Arrays.toString(options));
+        }
+        return new NodeNamePatternConverter(NODE_NAME.get());
+    }
+
+    private final String nodeName;
+
+    private NodeNamePatternConverter(String nodeName) {
+        super("NodeName", "node_name");
+        this.nodeName = nodeName;
+    }
+
+    @Override
+    public void format(LogEvent event, StringBuilder toAppendTo) {
+        toAppendTo.append(nodeName);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/rounding/Rounding.java b/server/src/main/java/org/elasticsearch/common/rounding/Rounding.java
index 93be4cc65e2b3..f8339079926fd 100644
--- a/server/src/main/java/org/elasticsearch/common/rounding/Rounding.java
+++ b/server/src/main/java/org/elasticsearch/common/rounding/Rounding.java
@@ -32,7 +32,10 @@
 
 /**
  * A strategy for rounding long values.
+ *
+ * Use the java based Rounding class where applicable
  */
+@Deprecated
 public abstract class Rounding implements Writeable {
 
     public abstract byte id();
@@ -404,7 +407,7 @@ public static void write(Rounding rounding, StreamOutput out) throws IOException
         }
 
         public static Rounding read(StreamInput in) throws IOException {
-            Rounding rounding = null;
+            Rounding rounding;
             byte id = in.readByte();
             switch (id) {
                 case TimeUnitRounding.ID: rounding = new TimeUnitRounding(in); break;
diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index 6f45a5d462bd8..137378f509d6f 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -155,6 +155,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         EngineConfig.INDEX_CODEC_SETTING,
         EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS,
         IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS,
+        IndexSettings.DEFAULT_PIPELINE,
 
         // validate that built-in similarities don't get redefined
         Setting.groupSetting("index.similarity.", (s) -> {
diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
index eee45743ee32e..e017e9e7ca93f 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
@@ -359,7 +359,7 @@ public void decrypt(char[] password) throws GeneralSecurityException, IOExceptio
             if (input.read() != -1) {
                 throw new SecurityException("Keystore has been corrupted or tampered with");
             }
-        } catch (EOFException e) {
+        } catch (IOException e) {
             throw new SecurityException("Keystore has been corrupted or tampered with", e);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/common/time/CompoundDateTimeFormatter.java b/server/src/main/java/org/elasticsearch/common/time/CompoundDateTimeFormatter.java
index df459679c22b4..0332c03814de7 100644
--- a/server/src/main/java/org/elasticsearch/common/time/CompoundDateTimeFormatter.java
+++ b/server/src/main/java/org/elasticsearch/common/time/CompoundDateTimeFormatter.java
@@ -20,8 +20,14 @@
 
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
 import java.time.format.DateTimeParseException;
 import java.time.temporal.TemporalAccessor;
+import java.time.temporal.TemporalField;
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.Map;
+import java.util.function.Consumer;
 
 /**
  * wrapper class around java.time.DateTimeFormatter that supports multiple formats for easier parsing,
@@ -29,6 +35,13 @@
  */
 public class CompoundDateTimeFormatter {
 
+    private static final Consumer<DateTimeFormatter[]> SAME_TIME_ZONE_VALIDATOR = (parsers) -> {
+        long distinctZones = Arrays.stream(parsers).map(DateTimeFormatter::getZone).distinct().count();
+        if (distinctZones > 1) {
+            throw new IllegalArgumentException("formatters must have the same time zone");
+        }
+    };
+
     final DateTimeFormatter printer;
     final DateTimeFormatter[] parsers;
 
@@ -36,6 +49,7 @@ public class CompoundDateTimeFormatter {
         if (parsers.length == 0) {
             throw new IllegalArgumentException("at least one date time formatter is required");
         }
+        SAME_TIME_ZONE_VALIDATOR.accept(parsers);
         this.printer = parsers[0];
         this.parsers = parsers;
     }
@@ -58,7 +72,18 @@ public TemporalAccessor parse(String input) {
         throw failure;
     }
 
+    /**
+     * Configure a specific time zone for a date formatter
+     *
+     * @param zoneId The zoneId this formatter shoulduse
+     * @return       The new formatter with all parsers switched to the specified timezone
+     */
     public CompoundDateTimeFormatter withZone(ZoneId zoneId) {
+        // shortcurt to not create new objects unnecessarily
+        if (zoneId.equals(parsers[0].getZone())) {
+            return this;
+        }
+
         final DateTimeFormatter[] parsersWithZone = new DateTimeFormatter[parsers.length];
         for (int i = 0; i < parsers.length; i++) {
             parsersWithZone[i] = parsers[i].withZone(zoneId);
@@ -67,7 +92,22 @@ public CompoundDateTimeFormatter withZone(ZoneId zoneId) {
         return new CompoundDateTimeFormatter(parsersWithZone);
     }
 
+    /**
+     * Configure defaults for missing values in a parser, then return a new compound date formatter
+     */
+    CompoundDateTimeFormatter parseDefaulting(Map<TemporalField, Long> fields) {
+        final DateTimeFormatter[] parsersWithDefaulting = new DateTimeFormatter[parsers.length];
+        for (int i = 0; i < parsers.length; i++) {
+            DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder().append(parsers[i]);
+            fields.forEach(builder::parseDefaulting);
+            parsersWithDefaulting[i] = builder.toFormatter(Locale.ROOT);
+        }
+
+        return new CompoundDateTimeFormatter(parsersWithDefaulting);
+    }
+
     public String format(TemporalAccessor accessor) {
         return printer.format(accessor);
     }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java
index eef2ab5558789..37efff5a0bebd 100644
--- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java
+++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java
@@ -52,30 +52,10 @@
 
 public class DateFormatters {
 
-    private static final DateTimeFormatter TIME_ZONE_FORMATTER_ZONE_ID = new DateTimeFormatterBuilder()
-        .appendZoneId()
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter TIME_ZONE_FORMATTER_WITHOUT_COLON = new DateTimeFormatterBuilder()
+    private static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON = new DateTimeFormatterBuilder()
         .appendOffset("+HHmm", "Z")
         .toFormatter(Locale.ROOT);
 
-    private static final DateTimeFormatter TIME_ZONE_FORMATTER_WITH_COLON = new DateTimeFormatterBuilder()
-        .appendOffset("+HH:mm", "Z")
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter TIME_ZONE_FORMATTER = new DateTimeFormatterBuilder()
-        .optionalStart().appendZoneId().optionalEnd()
-        .optionalStart().appendOffset("+HHmm", "Z").optionalEnd()
-        .optionalStart().appendOffset("+HH:mm", "Z").optionalEnd()
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter OPTIONAL_TIME_ZONE_FORMATTER = new DateTimeFormatterBuilder()
-        .optionalStart()
-        .append(TIME_ZONE_FORMATTER)
-        .optionalEnd()
-        .toFormatter(Locale.ROOT);
-
     private static final DateTimeFormatter STRICT_YEAR_MONTH_DAY_FORMATTER = new DateTimeFormatterBuilder()
         .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
         .appendLiteral("-")
@@ -101,7 +81,7 @@ public class DateFormatters {
         .appendFraction(MILLI_OF_SECOND, 3, 3, true)
         .optionalEnd()
         .optionalStart()
-        .append(TIME_ZONE_FORMATTER_WITHOUT_COLON)
+        .appendOffset("+HHmm", "Z")
         .optionalEnd()
         .optionalEnd()
         .toFormatter(Locale.ROOT);
@@ -115,483 +95,183 @@ public class DateFormatters {
         .appendFraction(MILLI_OF_SECOND, 3, 3, true)
         .optionalEnd()
         .optionalStart()
-        .append(TIME_ZONE_FORMATTER_WITH_COLON)
-        .optionalEnd()
-        .optionalEnd()
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter STRICT_DATE_OPTIONAL_TIME_FORMATTER_3 = new DateTimeFormatterBuilder()
-        .append(STRICT_YEAR_MONTH_DAY_FORMATTER)
-        .optionalStart()
-        .appendLiteral('T')
-        .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER)
-        .optionalStart()
-        .appendFraction(MILLI_OF_SECOND, 3, 3, true)
-        .optionalEnd()
-        .optionalStart()
-        .append(TIME_ZONE_FORMATTER_ZONE_ID)
+        .appendZoneOrOffsetId()
         .optionalEnd()
         .optionalEnd()
         .toFormatter(Locale.ROOT);
 
+    /**
+     * Returns a generic ISO datetime parser where the date is mandatory and the time is optional.
+     */
     private static final CompoundDateTimeFormatter STRICT_DATE_OPTIONAL_TIME =
-        new CompoundDateTimeFormatter(STRICT_DATE_OPTIONAL_TIME_FORMATTER_1, STRICT_DATE_OPTIONAL_TIME_FORMATTER_2,
-            STRICT_DATE_OPTIONAL_TIME_FORMATTER_3);
-
-    private static final DateTimeFormatter BASIC_TIME_NO_MILLIS_FORMATTER = new DateTimeFormatterBuilder()
+        new CompoundDateTimeFormatter(STRICT_DATE_OPTIONAL_TIME_FORMATTER_1, STRICT_DATE_OPTIONAL_TIME_FORMATTER_2);
+
+    /////////////////////////////////////////
+    //
+    // BEGIN basic time formatters
+    //
+    // these formatters to not have any splitting characters between hours, minutes, seconds, milliseconds
+    // this means they have to be strict with the exception of the last element
+    //
+    /////////////////////////////////////////
+
+    private static final DateTimeFormatter BASIC_TIME_NO_MILLIS_BASE = new DateTimeFormatterBuilder()
         .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE)
         .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE)
         .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE)
-        .append(OPTIONAL_TIME_ZONE_FORMATTER)
         .toFormatter(Locale.ROOT);
 
-    private static final CompoundDateTimeFormatter BASIC_TIME_NO_MILLIS = new CompoundDateTimeFormatter(BASIC_TIME_NO_MILLIS_FORMATTER);
+    /*
+     * Returns a basic formatter for a two digit hour of day, two digit minute
+     * of hour, two digit second of minute, and time zone offset (HHmmssZ).
+     */
+    private static final CompoundDateTimeFormatter BASIC_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(BASIC_TIME_NO_MILLIS_BASE).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(BASIC_TIME_NO_MILLIS_BASE).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
     private static final DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder()
         .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE)
         .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE)
         .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE)
         .appendFraction(MILLI_OF_SECOND, 1, 3, true)
-        .append(OPTIONAL_TIME_ZONE_FORMATTER)
         .toFormatter(Locale.ROOT);
 
-    private static final CompoundDateTimeFormatter BASIC_TIME = new CompoundDateTimeFormatter(BASIC_TIME_FORMATTER);
-
-    private static final DateTimeFormatter BASIC_T_TIME_FORMATTER = new DateTimeFormatterBuilder()
-        .appendLiteral("T")
-        .append(BASIC_TIME_FORMATTER)
+    private static final DateTimeFormatter BASIC_TIME_PRINTER = new DateTimeFormatterBuilder()
+        .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE)
+        .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE)
+        .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE)
+        .appendFraction(MILLI_OF_SECOND, 3, 3, true)
         .toFormatter(Locale.ROOT);
 
-    private static final CompoundDateTimeFormatter BASIC_T_TIME = new CompoundDateTimeFormatter(BASIC_T_TIME_FORMATTER);
+    /*
+     * Returns a basic formatter for a two digit hour of day, two digit minute
+     * of hour, two digit second of minute, three digit millis, and time zone
+     * offset (HHmmss.SSSZ).
+     */
+    private static final CompoundDateTimeFormatter BASIC_TIME = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(BASIC_TIME_PRINTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(BASIC_TIME_FORMATTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(BASIC_TIME_FORMATTER).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
-    private static final CompoundDateTimeFormatter BASIC_T_TIME_NO_MILLIS = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendLiteral("T")
-        .append(BASIC_TIME_NO_MILLIS_FORMATTER)
-        .toFormatter(Locale.ROOT));
+    private static final DateTimeFormatter BASIC_T_TIME_PRINTER =
+        new DateTimeFormatterBuilder().appendLiteral("T").append(BASIC_TIME_PRINTER).toFormatter(Locale.ROOT);
+
+    private static final DateTimeFormatter BASIC_T_TIME_FORMATTER =
+        new DateTimeFormatterBuilder().appendLiteral("T").append(BASIC_TIME_FORMATTER).toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a basic formatter for a two digit hour of day, two digit minute
+     * of hour, two digit second of minute, three digit millis, and time zone
+     * offset prefixed by 'T' ('T'HHmmss.SSSZ).
+     */
+    private static final CompoundDateTimeFormatter BASIC_T_TIME = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(BASIC_T_TIME_PRINTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(BASIC_T_TIME_FORMATTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(BASIC_T_TIME_FORMATTER).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
-    private static final CompoundDateTimeFormatter BASIC_DATE_TIME = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendValue(ChronoField.YEAR, 4, 4, SignStyle.NORMAL)
-        .appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NOT_NEGATIVE)
-        .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NOT_NEGATIVE)
-        .append(BASIC_T_TIME_FORMATTER)
-        .toFormatter(Locale.ROOT));
+    /*
+     * Returns a basic formatter for a two digit hour of day, two digit minute
+     * of hour, two digit second of minute, and time zone offset prefixed by 'T'
+     * ('T'HHmmssZ).
+     */
+    private static final CompoundDateTimeFormatter BASIC_T_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().appendLiteral("T").append(BASIC_TIME_NO_MILLIS_BASE)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().appendLiteral("T").append(BASIC_TIME_NO_MILLIS_BASE)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
-    private static final CompoundDateTimeFormatter BASIC_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
+    private static final DateTimeFormatter BASIC_YEAR_MONTH_DAY_FORMATTER = new DateTimeFormatterBuilder()
         .appendValue(ChronoField.YEAR, 4, 4, SignStyle.NORMAL)
         .appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NOT_NEGATIVE)
         .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral("T")
-        .append(BASIC_TIME_NO_MILLIS_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter BASIC_ORDINAL_DATE = new CompoundDateTimeFormatter(
-        DateTimeFormatter.ofPattern("yyyyDDD", Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter BASIC_ORDINAL_DATE_TIME = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendPattern("yyyyDDD")
-        .append(BASIC_T_TIME_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter BASIC_ORDINAL_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
-        new DateTimeFormatterBuilder()
-        .appendPattern("yyyyDDD")
-        .appendLiteral("T")
-        .append(BASIC_TIME_NO_MILLIS_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final DateTimeFormatter BASIC_WEEK_DATE_FORMATTER = new DateTimeFormatterBuilder()
-        .appendValue(IsoFields.WEEK_BASED_YEAR)
-        .appendLiteral("W")
-        .appendValue(IsoFields.WEEK_OF_WEEK_BASED_YEAR, 1, 2, SignStyle.NEVER)
-        .appendValue(ChronoField.DAY_OF_WEEK)
         .toFormatter(Locale.ROOT);
 
-    private static final CompoundDateTimeFormatter BASIC_WEEK_DATE = new CompoundDateTimeFormatter(BASIC_WEEK_DATE_FORMATTER);
-
-    private static final CompoundDateTimeFormatter BASIC_WEEK_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
-        new DateTimeFormatterBuilder()
-        .append(BASIC_WEEK_DATE_FORMATTER)
-        .appendLiteral("T")
-        .append(BASIC_TIME_NO_MILLIS_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter BASIC_WEEK_DATE_TIME = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .append(BASIC_WEEK_DATE_FORMATTER)
+    private static final DateTimeFormatter BASIC_DATE_TIME_FORMATTER = new DateTimeFormatterBuilder()
+        .append(BASIC_YEAR_MONTH_DAY_FORMATTER)
         .append(BASIC_T_TIME_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder()
-        .appendValue(ChronoField.YEAR, 1, 4, SignStyle.NORMAL)
-        .appendLiteral('-')
-        .appendValue(MONTH_OF_YEAR, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral('-')
-        .appendValue(DAY_OF_MONTH, 1, 2, SignStyle.NOT_NEGATIVE)
-        .toFormatter(Locale.ROOT);
-
-    private static final CompoundDateTimeFormatter DATE = new CompoundDateTimeFormatter(DATE_FORMATTER);
-
-    private static final CompoundDateTimeFormatter HOUR = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter DATE_HOUR = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .append(DATE_FORMATTER)
-        .appendLiteral("T")
-        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
-        .toFormatter(Locale.ROOT));
-
-    private static final DateTimeFormatter HOUR_MINUTE_FORMATTER = new DateTimeFormatterBuilder()
-        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
-        .toFormatter(Locale.ROOT);
-
-    private static final CompoundDateTimeFormatter HOUR_MINUTE = new CompoundDateTimeFormatter(HOUR_MINUTE_FORMATTER);
-
-    private static final DateTimeFormatter DATE_TIME_PREFIX = new DateTimeFormatterBuilder()
-        .append(DATE_FORMATTER)
-        .appendLiteral('T')
-        .append(HOUR_MINUTE_FORMATTER)
-        .optionalStart()
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .optionalEnd()
-        .toFormatter(Locale.ROOT);
-
-    // only the formatter, nothing optional here
-    private static final DateTimeFormatter DATE_TIME_NO_MILLIS_FORMATTER = new DateTimeFormatterBuilder()
-        .append(DATE_FORMATTER)
-        .appendLiteral('T')
-        .append(HOUR_MINUTE_FORMATTER)
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendZoneId()
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter DATE_TIME_NO_MILLIS_1 = new DateTimeFormatterBuilder()
-        .append(DATE_TIME_PREFIX)
-        .append(TIME_ZONE_FORMATTER_WITH_COLON)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter DATE_TIME_NO_MILLIS_2 = new DateTimeFormatterBuilder()
-        .append(DATE_TIME_PREFIX)
-        .append(TIME_ZONE_FORMATTER_WITHOUT_COLON)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter DATE_TIME_NO_MILLIS_3 = new DateTimeFormatterBuilder()
-        .append(DATE_TIME_PREFIX)
-        .append(TIME_ZONE_FORMATTER_ZONE_ID)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter DATE_TIME_NO_MILLIS_4 = new DateTimeFormatterBuilder()
-        .append(DATE_TIME_PREFIX)
-        .optionalStart()
-        .append(TIME_ZONE_FORMATTER_WITH_COLON)
-        .optionalEnd()
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter DATE_TIME_NO_MILLIS_5 = new DateTimeFormatterBuilder()
-        .append(DATE_TIME_PREFIX)
-        .optionalStart()
-        .append(TIME_ZONE_FORMATTER_WITHOUT_COLON)
-        .optionalEnd()
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter DATE_TIME_NO_MILLIS_6 = new DateTimeFormatterBuilder()
-        .append(DATE_TIME_PREFIX)
-        .optionalStart()
-        .append(TIME_ZONE_FORMATTER_ZONE_ID)
-        .optionalEnd()
-        .toFormatter(Locale.ROOT);
-
-    private static final CompoundDateTimeFormatter DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(DATE_TIME_NO_MILLIS_FORMATTER,
-        DATE_TIME_NO_MILLIS_1,  DATE_TIME_NO_MILLIS_2, DATE_TIME_NO_MILLIS_3, DATE_TIME_NO_MILLIS_4, DATE_TIME_NO_MILLIS_5,
-        DATE_TIME_NO_MILLIS_6);
-
-    private static final CompoundDateTimeFormatter DATE_TIME = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .append(DATE_FORMATTER)
-        .appendLiteral('T')
-        .append(HOUR_MINUTE_FORMATTER)
-        .optionalStart()
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
-        .optionalEnd()
-        .append(OPTIONAL_TIME_ZONE_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter DATE_OPTIONAL_TIME = new CompoundDateTimeFormatter(STRICT_DATE_OPTIONAL_TIME.printer,
-        new DateTimeFormatterBuilder()
-        .append(DATE_FORMATTER)
-        .parseLenient()
-        .optionalStart()
-        .appendLiteral('T')
-        .append(HOUR_MINUTE_FORMATTER)
-        .optionalStart()
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
-        .optionalEnd()
-        .append(OPTIONAL_TIME_ZONE_FORMATTER)
-        .optionalEnd()
-        .toFormatter(Locale.ROOT));
-
-    private static final DateTimeFormatter HOUR_MINUTE_SECOND_FORMATTER = new DateTimeFormatterBuilder()
-        .append(HOUR_MINUTE_FORMATTER)
-        .appendLiteral(":")
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .toFormatter(Locale.ROOT);
-
-    private static final CompoundDateTimeFormatter HOUR_MINUTE_SECOND = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .append(HOUR_MINUTE_FORMATTER)
-        .appendLiteral(":")
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter DATE_HOUR_MINUTE_SECOND = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .append(DATE_FORMATTER)
-        .appendLiteral("T")
-        .append(HOUR_MINUTE_SECOND_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter DATE_HOUR_MINUTE = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .append(DATE_FORMATTER)
-        .appendLiteral("T")
-        .append(HOUR_MINUTE_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final DateTimeFormatter HOUR_MINUTE_SECOND_MILLIS_FORMATTER = new DateTimeFormatterBuilder()
-        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
         .toFormatter(Locale.ROOT);
 
-    private static final CompoundDateTimeFormatter HOUR_MINUTE_SECOND_MILLIS =
-        new CompoundDateTimeFormatter(HOUR_MINUTE_SECOND_MILLIS_FORMATTER);
-
-    private static final CompoundDateTimeFormatter DATE_HOUR_MINUTE_SECOND_MILLIS =
-        new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .append(DATE_FORMATTER)
-        .appendLiteral("T")
-        .append(HOUR_MINUTE_SECOND_MILLIS_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter DATE_HOUR_MINUTE_SECOND_FRACTION =
-        new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .append(DATE_FORMATTER)
-        .appendLiteral("T")
-        .append(HOUR_MINUTE_SECOND_MILLIS_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final DateTimeFormatter ORDINAL_DATE_FORMATTER = new DateTimeFormatterBuilder()
-        .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
-        .appendLiteral('-')
-        .appendValue(DAY_OF_YEAR, 1, 3, SignStyle.NOT_NEGATIVE)
+    private static final DateTimeFormatter BASIC_DATE_TIME_PRINTER = new DateTimeFormatterBuilder()
+        .append(BASIC_YEAR_MONTH_DAY_FORMATTER)
+        .append(BASIC_T_TIME_PRINTER)
         .toFormatter(Locale.ROOT);
 
-    private static final CompoundDateTimeFormatter ORDINAL_DATE = new CompoundDateTimeFormatter(ORDINAL_DATE_FORMATTER);
-
-    private static final CompoundDateTimeFormatter ORDINAL_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
-        new DateTimeFormatterBuilder()
-        .append(ORDINAL_DATE_FORMATTER)
-        .appendLiteral('T')
-        .append(HOUR_MINUTE_SECOND_FORMATTER)
-        .append(OPTIONAL_TIME_ZONE_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter ORDINAL_DATE_TIME = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .append(ORDINAL_DATE_FORMATTER)
-        .appendLiteral('T')
-        .append(HOUR_MINUTE_FORMATTER)
-        .optionalStart()
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
-        .optionalEnd()
-        .append(TIME_ZONE_FORMATTER)
-        .toFormatter(Locale.ROOT));
-
-    private static final DateTimeFormatter TIME_FORMATTER_1 = new DateTimeFormatterBuilder()
-        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
-        .append(TIME_ZONE_FORMATTER_ZONE_ID)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter TIME_FORMATTER_2 = new DateTimeFormatterBuilder()
-        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
-        .append(TIME_ZONE_FORMATTER_WITH_COLON)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter TIME_FORMATTER_3 = new DateTimeFormatterBuilder()
-        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
-        .append(TIME_ZONE_FORMATTER_WITHOUT_COLON)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter TIME_PREFIX = new DateTimeFormatterBuilder()
-        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter TIME_ZONE_ID = new DateTimeFormatterBuilder()
-        .append(TIME_PREFIX)
-        .append(TIME_ZONE_FORMATTER_ZONE_ID)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter TIME_ZONE_WITH_COLON = new DateTimeFormatterBuilder()
-        .append(TIME_PREFIX)
-        .append(TIME_ZONE_FORMATTER_WITH_COLON)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter TIME_ZONE_WITHOUT_COLON = new DateTimeFormatterBuilder()
-        .append(TIME_PREFIX)
-        .append(TIME_ZONE_FORMATTER_WITHOUT_COLON)
-        .toFormatter(Locale.ROOT);
-
-    private static final CompoundDateTimeFormatter T_TIME = new CompoundDateTimeFormatter(
-        new DateTimeFormatterBuilder().appendLiteral("T").append(TIME_FORMATTER_1).toFormatter(Locale.ROOT),
-        new DateTimeFormatterBuilder().appendLiteral("T").append(TIME_FORMATTER_2).toFormatter(Locale.ROOT),
-        new DateTimeFormatterBuilder().appendLiteral("T").append(TIME_FORMATTER_3).toFormatter(Locale.ROOT)
+    /*
+     * Returns a basic formatter that combines a basic date and time, separated
+     * by a 'T' (yyyyMMdd'T'HHmmss.SSSZ).
+     */
+    private static final CompoundDateTimeFormatter BASIC_DATE_TIME = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(BASIC_DATE_TIME_PRINTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(BASIC_DATE_TIME_FORMATTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(BASIC_DATE_TIME_FORMATTER).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
     );
 
-    private static final DateTimeFormatter TIME_NO_MILLIS_FORMATTER_1 = new DateTimeFormatterBuilder()
-        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .append(TIME_ZONE_FORMATTER_ZONE_ID)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter TIME_NO_MILLIS_FORMATTER_2 = new DateTimeFormatterBuilder()
-        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .append(TIME_ZONE_FORMATTER_WITH_COLON)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter TIME_NO_MILLIS_FORMATTER_3 = new DateTimeFormatterBuilder()
-        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral(':')
-        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
-        .append(TIME_ZONE_FORMATTER_WITHOUT_COLON)
-        .toFormatter(Locale.ROOT);
-
-    private static final CompoundDateTimeFormatter TIME = new CompoundDateTimeFormatter(TIME_ZONE_ID, TIME_ZONE_WITH_COLON,
-        TIME_ZONE_WITHOUT_COLON);
-
-    private static final CompoundDateTimeFormatter TIME_NO_MILLIS =
-        new CompoundDateTimeFormatter(TIME_NO_MILLIS_FORMATTER_1, TIME_NO_MILLIS_FORMATTER_2, TIME_NO_MILLIS_FORMATTER_3);
-
-    private static final DateTimeFormatter T_TIME_NO_MILLIS_FORMATTER_1 = new DateTimeFormatterBuilder()
-        .appendLiteral("T")
-        .append(TIME_NO_MILLIS_FORMATTER_1)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter T_TIME_NO_MILLIS_FORMATTER_2 = new DateTimeFormatterBuilder()
-        .appendLiteral("T")
-        .append(TIME_NO_MILLIS_FORMATTER_2)
-        .toFormatter(Locale.ROOT);
-
-    private static final DateTimeFormatter T_TIME_NO_MILLIS_FORMATTER_3 = new DateTimeFormatterBuilder()
-        .appendLiteral("T")
-        .append(TIME_NO_MILLIS_FORMATTER_3)
-        .toFormatter(Locale.ROOT);
-
-    private static final CompoundDateTimeFormatter T_TIME_NO_MILLIS =
-        new CompoundDateTimeFormatter(T_TIME_NO_MILLIS_FORMATTER_1, T_TIME_NO_MILLIS_FORMATTER_2, T_TIME_NO_MILLIS_FORMATTER_3);
-
-    private static final DateTimeFormatter WEEK_DATE_FORMATTER = new DateTimeFormatterBuilder()
-        .appendValue(IsoFields.WEEK_BASED_YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
-        .appendLiteral("-W")
-        .appendValue(IsoFields.WEEK_OF_WEEK_BASED_YEAR, 1, 2, SignStyle.NOT_NEGATIVE)
-        .appendLiteral('-')
-        .appendValue(DAY_OF_WEEK, 1)
-        .toFormatter(Locale.ROOT);
-
-    private static final CompoundDateTimeFormatter WEEK_DATE = new CompoundDateTimeFormatter(WEEK_DATE_FORMATTER);
-
-    private static final CompoundDateTimeFormatter WEEK_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
-        new DateTimeFormatterBuilder().append(WEEK_DATE_FORMATTER).append(T_TIME_NO_MILLIS_FORMATTER_1).toFormatter(Locale.ROOT),
-        new DateTimeFormatterBuilder().append(WEEK_DATE_FORMATTER).append(T_TIME_NO_MILLIS_FORMATTER_2).toFormatter(Locale.ROOT),
-        new DateTimeFormatterBuilder().append(WEEK_DATE_FORMATTER).append(T_TIME_NO_MILLIS_FORMATTER_3).toFormatter(Locale.ROOT)
-        );
-
-    private static final CompoundDateTimeFormatter WEEK_DATE_TIME = new CompoundDateTimeFormatter(
-        new DateTimeFormatterBuilder().append(WEEK_DATE_FORMATTER).appendLiteral("T").append(TIME_FORMATTER_1).toFormatter(Locale.ROOT),
-        new DateTimeFormatterBuilder().append(WEEK_DATE_FORMATTER).appendLiteral("T").append(TIME_FORMATTER_2).toFormatter(Locale.ROOT),
-        new DateTimeFormatterBuilder().append(WEEK_DATE_FORMATTER).appendLiteral("T").append(TIME_FORMATTER_3).toFormatter(Locale.ROOT)
+    private static final DateTimeFormatter BASIC_DATE_T =
+        new DateTimeFormatterBuilder().append(BASIC_YEAR_MONTH_DAY_FORMATTER).appendLiteral("T").toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a basic formatter that combines a basic date and time without millis,
+     * separated by a 'T' (yyyyMMdd'T'HHmmssZ).
+     */
+    private static final CompoundDateTimeFormatter BASIC_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(BASIC_DATE_T).append(BASIC_TIME_NO_MILLIS_BASE)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(BASIC_DATE_T).append(BASIC_TIME_NO_MILLIS_BASE)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
     );
 
-    private static final CompoundDateTimeFormatter WEEK_YEAR = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendValue(WeekFields.ISO.weekBasedYear())
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter WEEKYEAR_WEEK = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendValue(WeekFields.ISO.weekBasedYear())
-        .appendLiteral("-W")
-        .appendValue(WeekFields.ISO.weekOfWeekBasedYear())
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter WEEKYEAR_WEEK_DAY = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendValue(WeekFields.ISO.weekBasedYear())
-        .appendLiteral("-W")
-        .appendValue(WeekFields.ISO.weekOfWeekBasedYear())
-        .appendLiteral("-")
-        .appendValue(WeekFields.ISO.dayOfWeek())
-        .toFormatter(Locale.ROOT));
-
-    private static final CompoundDateTimeFormatter YEAR = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendValue(ChronoField.YEAR)
-        .toFormatter(Locale.ROOT));
+    /*
+     * Returns a formatter for a full ordinal date, using a four
+     * digit year and three digit dayOfYear (yyyyDDD).
+     */
+    private static final CompoundDateTimeFormatter BASIC_ORDINAL_DATE = new CompoundDateTimeFormatter(
+        DateTimeFormatter.ofPattern("yyyyDDD", Locale.ROOT));
 
-    private static final CompoundDateTimeFormatter YEAR_MONTH = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendValue(ChronoField.YEAR)
-        .appendLiteral("-")
-        .appendValue(MONTH_OF_YEAR)
-        .toFormatter(Locale.ROOT));
+    /*
+     * Returns a formatter for a full ordinal date and time, using a four
+     * digit year and three digit dayOfYear (yyyyDDD'T'HHmmss.SSSZ).
+     */
+    private static final CompoundDateTimeFormatter BASIC_ORDINAL_DATE_TIME = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().appendPattern("yyyyDDD").append(BASIC_T_TIME_PRINTER)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().appendPattern("yyyyDDD").append(BASIC_T_TIME_FORMATTER)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
 
-    private static final CompoundDateTimeFormatter YEAR_MONTH_DAY = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendValue(ChronoField.YEAR)
-        .appendLiteral("-")
-        .appendValue(MONTH_OF_YEAR)
-        .appendLiteral("-")
-        .appendValue(DAY_OF_MONTH)
-        .toFormatter(Locale.ROOT));
+    );
 
-    private static final CompoundDateTimeFormatter EPOCH_SECOND = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendValue(ChronoField.INSTANT_SECONDS)
-        .toFormatter(Locale.ROOT));
+    /*
+     * Returns a formatter for a full ordinal date and time without millis,
+     * using a four digit year and three digit dayOfYear (yyyyDDD'T'HHmmssZ).
+     */
+    private static final CompoundDateTimeFormatter BASIC_ORDINAL_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().appendPattern("yyyyDDD").appendLiteral("T").append(BASIC_TIME_NO_MILLIS_BASE)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().appendPattern("yyyyDDD").appendLiteral("T").append(BASIC_TIME_NO_MILLIS_BASE)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
-    private static final CompoundDateTimeFormatter EPOCH_MILLIS = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .appendValue(ChronoField.INSTANT_SECONDS, 1, 19, SignStyle.NEVER)
-        .appendValue(ChronoField.MILLI_OF_SECOND, 3)
-        .toFormatter(Locale.ROOT));
+    private static final DateTimeFormatter BASIC_WEEK_DATE_FORMATTER = new DateTimeFormatterBuilder()
+        .appendValue(IsoFields.WEEK_BASED_YEAR)
+        .appendLiteral("W")
+        .appendValue(IsoFields.WEEK_OF_WEEK_BASED_YEAR, 1, 2, SignStyle.NEVER)
+        .appendValue(ChronoField.DAY_OF_WEEK)
+        .toFormatter(Locale.ROOT);
 
+    /////////////////////////////////////////
+    //
+    // END basic time formatters
+    //
+    /////////////////////////////////////////
+
+    /////////////////////////////////////////
+    //
+    // start strict formatters
+    //
+    /////////////////////////////////////////
     private static final DateTimeFormatter STRICT_BASIC_WEEK_DATE_FORMATTER = new DateTimeFormatterBuilder()
         .parseStrict()
         .appendValue(IsoFields.WEEK_BASED_YEAR, 4)
@@ -600,97 +280,208 @@ public class DateFormatters {
         .appendValue(ChronoField.DAY_OF_WEEK)
         .toFormatter(Locale.ROOT);
 
-    private static final CompoundDateTimeFormatter STRICT_BASIC_WEEK_DATE = new CompoundDateTimeFormatter(STRICT_BASIC_WEEK_DATE_FORMATTER);
+    private static final DateTimeFormatter STRICT_BASIC_WEEK_DATE_PRINTER = new DateTimeFormatterBuilder()
+        .parseStrict()
+        .appendValue(IsoFields.WEEK_BASED_YEAR, 4)
+        .appendLiteral("W")
+        .appendValue(IsoFields.WEEK_OF_WEEK_BASED_YEAR, 2, 2, SignStyle.NEVER)
+        .appendValue(ChronoField.DAY_OF_WEEK)
+        .toFormatter(Locale.ROOT);
 
+    /*
+     * Returns a basic formatter for a full date as four digit weekyear, two
+     * digit week of weekyear, and one digit day of week (xxxx'W'wwe).
+     */
+    private static final CompoundDateTimeFormatter STRICT_BASIC_WEEK_DATE =
+        new CompoundDateTimeFormatter(STRICT_BASIC_WEEK_DATE_PRINTER, STRICT_BASIC_WEEK_DATE_FORMATTER);
+
+    /*
+     * Returns a basic formatter that combines a basic weekyear date and time
+     * without millis, separated by a 'T' (xxxx'W'wwe'T'HHmmssX).
+     */
     private static final CompoundDateTimeFormatter STRICT_BASIC_WEEK_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
         new DateTimeFormatterBuilder()
-        .append(STRICT_BASIC_WEEK_DATE_FORMATTER)
-        .append(DateTimeFormatter.ofPattern("'T'HHmmssX", Locale.ROOT))
-        .toFormatter(Locale.ROOT));
+            .append(STRICT_BASIC_WEEK_DATE_PRINTER).append(DateTimeFormatter.ofPattern("'T'HHmmssX", Locale.ROOT))
+            .toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder()
+            .append(STRICT_BASIC_WEEK_DATE_FORMATTER).append(DateTimeFormatter.ofPattern("'T'HHmmssX", Locale.ROOT))
+            .toFormatter(Locale.ROOT)
+    );
 
+    /*
+     * Returns a basic formatter that combines a basic weekyear date and time,
+     * separated by a 'T' (xxxx'W'wwe'T'HHmmss.SSSX).
+     */
     private static final CompoundDateTimeFormatter STRICT_BASIC_WEEK_DATE_TIME = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder()
+        .append(STRICT_BASIC_WEEK_DATE_PRINTER)
+        .append(DateTimeFormatter.ofPattern("'T'HHmmss.SSSX", Locale.ROOT))
+        .toFormatter(Locale.ROOT),
         new DateTimeFormatterBuilder()
         .append(STRICT_BASIC_WEEK_DATE_FORMATTER)
         .append(DateTimeFormatter.ofPattern("'T'HHmmss.SSSX", Locale.ROOT))
-        .toFormatter(Locale.ROOT));
+        .toFormatter(Locale.ROOT)
+    );
 
+    /*
+     * An ISO date formatter that formats or parses a date without an offset, such as '2011-12-03'.
+     */
     private static final CompoundDateTimeFormatter STRICT_DATE = new CompoundDateTimeFormatter(
         DateTimeFormatter.ISO_LOCAL_DATE.withResolverStyle(ResolverStyle.LENIENT));
 
+    /*
+     * A date formatter that formats or parses a date plus an hour without an offset, such as '2011-12-03T01'.
+     */
     private static final CompoundDateTimeFormatter STRICT_DATE_HOUR = new CompoundDateTimeFormatter(
         DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH", Locale.ROOT));
 
+    /*
+     * A date formatter that formats or parses a date plus an hour/minute without an offset, such as '2011-12-03T01:10'.
+     */
     private static final CompoundDateTimeFormatter STRICT_DATE_HOUR_MINUTE = new CompoundDateTimeFormatter(
         DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm", Locale.ROOT));
 
+    /*
+     * A strict date formatter that formats or parses a date without an offset, such as '2011-12-03'.
+     */
     private static final CompoundDateTimeFormatter STRICT_YEAR_MONTH_DAY = new CompoundDateTimeFormatter(STRICT_YEAR_MONTH_DAY_FORMATTER);
 
+    /*
+     * A strict formatter that formats or parses a year and a month, such as '2011-12'.
+     */
     private static final CompoundDateTimeFormatter STRICT_YEAR_MONTH = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
         .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
         .appendLiteral("-")
         .appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NOT_NEGATIVE)
         .toFormatter(Locale.ROOT));
 
+    /*
+     * A strict formatter that formats or parses a year, such as '2011'.
+     */
     private static final CompoundDateTimeFormatter STRICT_YEAR = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
         .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
         .toFormatter(Locale.ROOT));
 
+    /*
+     * A strict formatter that formats or parses a hour, minute and second, such as '09:43:25'.
+     */
     private static final CompoundDateTimeFormatter STRICT_HOUR_MINUTE_SECOND =
         new CompoundDateTimeFormatter(STRICT_HOUR_MINUTE_SECOND_FORMATTER);
 
-    private static final CompoundDateTimeFormatter STRICT_DATE_TIME = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
+    private static final DateTimeFormatter STRICT_DATE_FORMATTER = new DateTimeFormatterBuilder()
         .append(STRICT_YEAR_MONTH_DAY_FORMATTER)
         .appendLiteral('T')
         .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER)
         .optionalStart()
         .appendFraction(MILLI_OF_SECOND, 3, 3, true)
         .optionalEnd()
-        .append(OPTIONAL_TIME_ZONE_FORMATTER)
-        .toFormatter(Locale.ROOT));
+        .toFormatter(Locale.ROOT);
 
-    private static final CompoundDateTimeFormatter STRICT_ORDINAL_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
-        new DateTimeFormatterBuilder()
+    /*
+     * Returns a formatter that combines a full date and time, separated by a 'T'
+     * (yyyy-MM-dd'T'HH:mm:ss.SSSZZ).
+     */
+    private static final CompoundDateTimeFormatter STRICT_DATE_TIME = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(STRICT_DATE_FORMATTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(STRICT_DATE_FORMATTER).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    private static final DateTimeFormatter STRICT_ORDINAL_DATE_TIME_NO_MILLIS_BASE = new DateTimeFormatterBuilder()
         .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
         .appendLiteral('-')
         .appendValue(DAY_OF_YEAR, 3, 3, SignStyle.NOT_NEGATIVE)
         .appendLiteral('T')
         .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER)
-        .append(OPTIONAL_TIME_ZONE_FORMATTER)
-        .toFormatter(Locale.ROOT));
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter for a full ordinal date and time without millis,
+     * using a four digit year and three digit dayOfYear (yyyy-DDD'T'HH:mm:ssZZ).
+     */
+    private static final CompoundDateTimeFormatter STRICT_ORDINAL_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(STRICT_ORDINAL_DATE_TIME_NO_MILLIS_BASE)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(STRICT_ORDINAL_DATE_TIME_NO_MILLIS_BASE)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
-    private static final CompoundDateTimeFormatter STRICT_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
+    private static final DateTimeFormatter STRICT_DATE_TIME_NO_MILLIS_FORMATTER = new DateTimeFormatterBuilder()
         .append(STRICT_YEAR_MONTH_DAY_FORMATTER)
         .appendLiteral('T')
         .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER)
-        .append(OPTIONAL_TIME_ZONE_FORMATTER)
-        .toFormatter(Locale.ROOT));
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter that combines a full date and time without millis,
+     * separated by a 'T' (yyyy-MM-dd'T'HH:mm:ssZZ).
+     */
+    private static final CompoundDateTimeFormatter STRICT_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(STRICT_DATE_TIME_NO_MILLIS_FORMATTER)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(STRICT_DATE_TIME_NO_MILLIS_FORMATTER)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
+    // NOTE: this is not a strict formatter to retain the joda time based behaviour, even though it's named like this
     private static final DateTimeFormatter STRICT_HOUR_MINUTE_SECOND_MILLIS_FORMATTER = new DateTimeFormatterBuilder()
         .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER)
         .appendFraction(MILLI_OF_SECOND, 1, 3, true)
         .toFormatter(Locale.ROOT);
 
+    private static final DateTimeFormatter STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER = new DateTimeFormatterBuilder()
+        .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER)
+        .appendFraction(MILLI_OF_SECOND, 3, 3, true)
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter for a two digit hour of day, two digit minute of
+     * hour, two digit second of minute, and three digit fraction of
+     * second (HH:mm:ss.SSS).
+     *
+     * NOTE: this is not a strict formatter to retain the joda time based behaviour,
+     *       even though it's named like this
+     */
     private static final CompoundDateTimeFormatter STRICT_HOUR_MINUTE_SECOND_MILLIS =
-        new CompoundDateTimeFormatter(STRICT_HOUR_MINUTE_SECOND_MILLIS_FORMATTER);
+        new CompoundDateTimeFormatter(STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER, STRICT_HOUR_MINUTE_SECOND_MILLIS_FORMATTER);
 
     private static final CompoundDateTimeFormatter STRICT_HOUR_MINUTE_SECOND_FRACTION = STRICT_HOUR_MINUTE_SECOND_MILLIS;
 
+    /*
+     * Returns a formatter that combines a full date, two digit hour of day,
+     * two digit minute of hour, two digit second of minute, and three digit
+     * fraction of second (yyyy-MM-dd'T'HH:mm:ss.SSS).
+     */
     private static final CompoundDateTimeFormatter STRICT_DATE_HOUR_MINUTE_SECOND_FRACTION = new CompoundDateTimeFormatter(
         new DateTimeFormatterBuilder()
-        .append(STRICT_YEAR_MONTH_DAY_FORMATTER)
-        .appendLiteral("T")
-        .append(STRICT_HOUR_MINUTE_SECOND_MILLIS_FORMATTER)
-        .toFormatter(Locale.ROOT));
+            .append(STRICT_YEAR_MONTH_DAY_FORMATTER)
+            .appendLiteral("T")
+            .append(STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER)
+            .toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder()
+            .append(STRICT_YEAR_MONTH_DAY_FORMATTER)
+            .appendLiteral("T")
+            .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER)
+            //  this one here is lenient as well to retain joda time based bwc compatibility
+            .appendFraction(MILLI_OF_SECOND, 1, 3, true)
+            .toFormatter(Locale.ROOT)
+    );
 
     private static final CompoundDateTimeFormatter STRICT_DATE_HOUR_MINUTE_SECOND_MILLIS = STRICT_DATE_HOUR_MINUTE_SECOND_FRACTION;
 
+    /*
+     * Returns a formatter for a two digit hour of day. (HH)
+     */
     private static final CompoundDateTimeFormatter STRICT_HOUR =
         new CompoundDateTimeFormatter(DateTimeFormatter.ofPattern("HH", Locale.ROOT));
 
+    /*
+     * Returns a formatter for a two digit hour of day and two digit minute of
+     * hour. (HH:mm)
+     */
     private static final CompoundDateTimeFormatter STRICT_HOUR_MINUTE =
         new CompoundDateTimeFormatter(DateTimeFormatter.ofPattern("HH:mm", Locale.ROOT));
 
-    private static final CompoundDateTimeFormatter STRICT_ORDINAL_DATE_TIME = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
+    private static final DateTimeFormatter STRICT_ORDINAL_DATE_TIME_FORMATTER_BASE = new DateTimeFormatterBuilder()
         .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
         .appendLiteral('-')
         .appendValue(DAY_OF_YEAR, 3, 3, SignStyle.NOT_NEGATIVE)
@@ -699,62 +490,139 @@ public class DateFormatters {
         .optionalStart()
         .appendLiteral(':')
         .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE)
-        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
+        .appendFraction(MILLI_OF_SECOND, 3, 3, true)
         .optionalEnd()
-        .append(OPTIONAL_TIME_ZONE_FORMATTER)
-        .toFormatter(Locale.ROOT));
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter for a full ordinal date and time, using a four
+     * digit year and three digit dayOfYear (yyyy-DDD'T'HH:mm:ss.SSSZZ).
+     */
+    private static final CompoundDateTimeFormatter STRICT_ORDINAL_DATE_TIME = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(STRICT_ORDINAL_DATE_TIME_FORMATTER_BASE)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(STRICT_ORDINAL_DATE_TIME_FORMATTER_BASE)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
-    private static final DateTimeFormatter STRICT_TIME_FORMATTER = new DateTimeFormatterBuilder()
+    // Note: milliseconds parsing is not strict, others are
+    private static final DateTimeFormatter STRICT_TIME_FORMATTER_BASE = new DateTimeFormatterBuilder()
         .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE)
         .appendLiteral(':')
         .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE)
         .appendLiteral(':')
         .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE)
         .appendFraction(MILLI_OF_SECOND, 1, 3, true)
-        .append(TIME_ZONE_FORMATTER)
-        .toFormatter(Locale.ROOT);
-
-    private static final CompoundDateTimeFormatter STRICT_TIME = new CompoundDateTimeFormatter(STRICT_TIME_FORMATTER);
-
-    private static final DateTimeFormatter STRICT_T_TIME_FORMATTER = new DateTimeFormatterBuilder()
-        .appendLiteral("T")
-        .append(STRICT_TIME_FORMATTER)
         .toFormatter(Locale.ROOT);
 
-    private static final CompoundDateTimeFormatter STRICT_T_TIME = new CompoundDateTimeFormatter(STRICT_T_TIME_FORMATTER);
-
-    private static final DateTimeFormatter STRICT_TIME_NO_MILLIS_FORMATTER = new DateTimeFormatterBuilder()
+    private static final DateTimeFormatter STRICT_TIME_PRINTER = new DateTimeFormatterBuilder()
         .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE)
         .appendLiteral(':')
         .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE)
         .appendLiteral(':')
         .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE)
-        .append(TIME_ZONE_FORMATTER)
+        .appendFraction(MILLI_OF_SECOND, 3, 3, true)
         .toFormatter(Locale.ROOT);
 
-    private static final CompoundDateTimeFormatter STRICT_TIME_NO_MILLIS = new CompoundDateTimeFormatter(STRICT_TIME_NO_MILLIS_FORMATTER);
+    /*
+     * Returns a formatter for a two digit hour of day, two digit minute of
+     * hour, two digit second of minute, three digit fraction of second, and
+     * time zone offset (HH:mm:ss.SSSZZ).
+     */
+    private static final CompoundDateTimeFormatter STRICT_TIME = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(STRICT_TIME_PRINTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(STRICT_TIME_FORMATTER_BASE).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(STRICT_TIME_FORMATTER_BASE).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a formatter for a two digit hour of day, two digit minute of
+     * hour, two digit second of minute, three digit fraction of second, and
+     * time zone offset prefixed by 'T' ('T'HH:mm:ss.SSSZZ).
+     */
+    private static final CompoundDateTimeFormatter STRICT_T_TIME = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().appendLiteral('T').append(STRICT_TIME_PRINTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().appendLiteral('T').append(STRICT_TIME_FORMATTER_BASE)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().appendLiteral('T').append(STRICT_TIME_FORMATTER_BASE)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
-    private static final DateTimeFormatter STRICT_T_TIME_NO_MILLIS_FORMATTER = new DateTimeFormatterBuilder()
-        .appendLiteral("T")
-        .append(STRICT_TIME_NO_MILLIS_FORMATTER)
+    private static final DateTimeFormatter STRICT_TIME_NO_MILLIS_BASE = new DateTimeFormatterBuilder()
+        .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral(':')
+        .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral(':')
+        .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE)
         .toFormatter(Locale.ROOT);
 
-    private static final CompoundDateTimeFormatter STRICT_T_TIME_NO_MILLIS =
-        new CompoundDateTimeFormatter(STRICT_T_TIME_NO_MILLIS_FORMATTER);
+    /*
+     * Returns a formatter for a two digit hour of day, two digit minute of
+     * hour, two digit second of minute, and time zone offset (HH:mm:ssZZ).
+     */
+    private static final CompoundDateTimeFormatter STRICT_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(STRICT_TIME_NO_MILLIS_BASE).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(STRICT_TIME_NO_MILLIS_BASE).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
-    private static final CompoundDateTimeFormatter STRICT_WEEK_DATE = new CompoundDateTimeFormatter(DateTimeFormatter.ISO_WEEK_DATE);
+    /*
+     * Returns a formatter for a two digit hour of day, two digit minute of
+     * hour, two digit second of minute, and time zone offset prefixed
+     * by 'T' ('T'HH:mm:ssZZ).
+     */
+    private static final CompoundDateTimeFormatter STRICT_T_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().appendLiteral("T").append(STRICT_TIME_NO_MILLIS_BASE)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().appendLiteral("T").append(STRICT_TIME_NO_MILLIS_BASE)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
+    private static final DateTimeFormatter ISO_WEEK_DATE = new DateTimeFormatterBuilder()
+            .parseCaseInsensitive()
+            .appendValue(IsoFields.WEEK_BASED_YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
+            .appendLiteral("-W")
+            .appendValue(IsoFields.WEEK_OF_WEEK_BASED_YEAR, 2)
+            .appendLiteral('-')
+            .appendValue(DAY_OF_WEEK, 1)
+            .toFormatter(Locale.ROOT);
+
+    private static final DateTimeFormatter ISO_WEEK_DATE_T = new DateTimeFormatterBuilder()
+            .append(ISO_WEEK_DATE)
+            .appendLiteral('T')
+            .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter for a full date as four digit weekyear, two digit
+     * week of weekyear, and one digit day of week (xxxx-'W'ww-e).
+     */
+    private static final CompoundDateTimeFormatter STRICT_WEEK_DATE = new CompoundDateTimeFormatter(ISO_WEEK_DATE);
+
+    /*
+     * Returns a formatter that combines a full weekyear date and time without millis,
+     * separated by a 'T' (xxxx-'W'ww-e'T'HH:mm:ssZZ).
+     */
     private static final CompoundDateTimeFormatter STRICT_WEEK_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
-        new DateTimeFormatterBuilder()
-        .append(DateTimeFormatter.ISO_WEEK_DATE)
-        .append(STRICT_T_TIME_NO_MILLIS_FORMATTER)
-        .toFormatter(Locale.ROOT));
+        new DateTimeFormatterBuilder().append(ISO_WEEK_DATE_T)
+            .append(STRICT_TIME_NO_MILLIS_BASE).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(ISO_WEEK_DATE_T)
+            .append(STRICT_TIME_NO_MILLIS_BASE).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
-    private static final CompoundDateTimeFormatter STRICT_WEEK_DATE_TIME = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
-        .append(DateTimeFormatter.ISO_WEEK_DATE)
-        .append(STRICT_T_TIME_FORMATTER)
-        .toFormatter(Locale.ROOT));
+    /*
+     * Returns a formatter that combines a full weekyear date and time,
+     * separated by a 'T' (xxxx-'W'ww-e'T'HH:mm:ss.SSSZZ).
+     */
+    private static final CompoundDateTimeFormatter STRICT_WEEK_DATE_TIME = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().append(ISO_WEEK_DATE_T).append(STRICT_TIME_PRINTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(ISO_WEEK_DATE_T).append(STRICT_TIME_FORMATTER_BASE)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(ISO_WEEK_DATE_T).append(STRICT_TIME_FORMATTER_BASE)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
 
+    /*
+     * Returns a formatter for a four digit weekyear
+     */
     private static final CompoundDateTimeFormatter STRICT_WEEKYEAR = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
         .appendValue(WeekFields.ISO.weekBasedYear(), 4, 10, SignStyle.EXCEEDS_PAD)
         .toFormatter(Locale.ROOT));
@@ -765,19 +633,576 @@ public class DateFormatters {
         .appendValue(WeekFields.ISO.weekOfWeekBasedYear(), 2, 2, SignStyle.NOT_NEGATIVE)
         .toFormatter(Locale.ROOT);
 
+    /*
+     * Returns a formatter for a four digit weekyear and two digit week of
+     * weekyear. (xxxx-'W'ww)
+     */
     private static final CompoundDateTimeFormatter STRICT_WEEKYEAR_WEEK = new CompoundDateTimeFormatter(STRICT_WEEKYEAR_WEEK_FORMATTER);
 
+    /*
+     * Returns a formatter for a four digit weekyear, two digit week of
+     * weekyear, and one digit day of week. (xxxx-'W'ww-e)
+     */
     private static final CompoundDateTimeFormatter STRICT_WEEKYEAR_WEEK_DAY = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
         .append(STRICT_WEEKYEAR_WEEK_FORMATTER)
         .appendLiteral("-")
         .appendValue(WeekFields.ISO.dayOfWeek())
         .toFormatter(Locale.ROOT));
 
-    private static final CompoundDateTimeFormatter BASIC_ISO_DATE = new CompoundDateTimeFormatter(DateTimeFormatter.BASIC_ISO_DATE);
-    private static final CompoundDateTimeFormatter ISO_ORDINAL_DATE = new CompoundDateTimeFormatter(DateTimeFormatter.ISO_ORDINAL_DATE);
+    /*
+     * Returns a formatter that combines a full date, two digit hour of day,
+     * two digit minute of hour, and two digit second of
+     * minute. (yyyy-MM-dd'T'HH:mm:ss)
+     */
     private static final CompoundDateTimeFormatter STRICT_DATE_HOUR_MINUTE_SECOND =
         new CompoundDateTimeFormatter(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT));
 
+    /*
+     * A basic formatter for a full date as four digit year, two digit
+     * month of year, and two digit day of month (yyyyMMdd).
+     */
+    private static final CompoundDateTimeFormatter BASIC_DATE = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder()
+            .appendValue(ChronoField.YEAR, 4, 4, SignStyle.NORMAL)
+            .appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NOT_NEGATIVE)
+            .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NOT_NEGATIVE)
+            .toFormatter(Locale.ROOT).withZone(ZoneOffset.UTC),
+        new DateTimeFormatterBuilder()
+            .appendValue(ChronoField.YEAR, 1, 4, SignStyle.NORMAL)
+            .appendValue(MONTH_OF_YEAR, 1, 2, SignStyle.NOT_NEGATIVE)
+            .appendValue(DAY_OF_MONTH, 1, 2, SignStyle.NOT_NEGATIVE)
+            .toFormatter(Locale.ROOT).withZone(ZoneOffset.UTC)
+    );
+
+    private static final DateTimeFormatter STRICT_ORDINAL_DATE_FORMATTER = new DateTimeFormatterBuilder()
+        .parseCaseInsensitive()
+        .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
+        .appendLiteral('-')
+        .appendValue(DAY_OF_YEAR, 3)
+        .optionalStart()
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter for a full ordinal date, using a four
+     * digit year and three digit dayOfYear (yyyy-DDD).
+     */
+    private static final CompoundDateTimeFormatter STRICT_ORDINAL_DATE = new CompoundDateTimeFormatter(STRICT_ORDINAL_DATE_FORMATTER);
+
+    /////////////////////////////////////////
+    //
+    // end strict formatters
+    //
+    /////////////////////////////////////////
+
+    /////////////////////////////////////////
+    //
+    // start lenient formatters
+    //
+    /////////////////////////////////////////
+
+    private static final DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder()
+        .appendValue(ChronoField.YEAR, 1, 5, SignStyle.NORMAL)
+        .optionalStart()
+        .appendLiteral('-')
+        .appendValue(MONTH_OF_YEAR, 1, 2, SignStyle.NOT_NEGATIVE)
+        .optionalStart()
+        .appendLiteral('-')
+        .appendValue(DAY_OF_MONTH, 1, 2, SignStyle.NOT_NEGATIVE)
+        .optionalEnd()
+        .optionalEnd()
+        .toFormatter(Locale.ROOT);
+
+    private static final DateTimeFormatter HOUR_MINUTE_FORMATTER = new DateTimeFormatterBuilder()
+        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral(':')
+        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * a date formatter with optional time, being very lenient, format is
+     * yyyy-MM-dd'T'HH:mm:ss.SSSZ
+     */
+    private static final CompoundDateTimeFormatter DATE_OPTIONAL_TIME = new CompoundDateTimeFormatter(STRICT_DATE_OPTIONAL_TIME.printer,
+        new DateTimeFormatterBuilder()
+            .append(DATE_FORMATTER)
+            .optionalStart()
+            .appendLiteral('T')
+            .optionalStart()
+            .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
+            .optionalStart()
+            .appendLiteral(':')
+            .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
+            .optionalStart()
+            .appendLiteral(':')
+            .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
+            .optionalEnd()
+            .optionalStart()
+            .appendFraction(MILLI_OF_SECOND, 1, 3, true)
+            .optionalEnd()
+            .optionalStart().appendZoneOrOffsetId().optionalEnd()
+            .optionalEnd()
+            .optionalEnd()
+            .optionalEnd()
+            .toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder()
+            .append(DATE_FORMATTER)
+            .optionalStart()
+            .appendLiteral('T')
+            .optionalStart()
+            .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
+            .optionalStart()
+            .appendLiteral(':')
+            .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
+            .optionalStart()
+            .appendLiteral(':')
+            .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
+            .optionalEnd()
+            .optionalStart()
+            .appendFraction(MILLI_OF_SECOND, 1, 3, true)
+            .optionalEnd()
+            .optionalStart().appendOffset("+HHmm", "Z").optionalEnd()
+            .optionalEnd()
+            .optionalEnd()
+            .optionalEnd()
+            .toFormatter(Locale.ROOT));
+
+    private static final DateTimeFormatter HOUR_MINUTE_SECOND_FORMATTER = new DateTimeFormatterBuilder()
+        .append(HOUR_MINUTE_FORMATTER)
+        .appendLiteral(":")
+        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
+        .toFormatter(Locale.ROOT);
+
+    private static final DateTimeFormatter HOUR_MINUTE_SECOND_MILLIS_FORMATTER = new DateTimeFormatterBuilder()
+        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral(':')
+        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral(':')
+        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
+        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
+        .toFormatter(Locale.ROOT);
+
+    private static final DateTimeFormatter ORDINAL_DATE_FORMATTER = new DateTimeFormatterBuilder()
+        .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
+        .appendLiteral('-')
+        .appendValue(DAY_OF_YEAR, 1, 3, SignStyle.NOT_NEGATIVE)
+        .toFormatter(Locale.ROOT);
+
+    private static final DateTimeFormatter ORDINAL_DATE_PRINTER = new DateTimeFormatterBuilder()
+        .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
+        .appendLiteral('-')
+        .appendValue(DAY_OF_YEAR, 3, 3, SignStyle.NOT_NEGATIVE)
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter for a full ordinal date, using a four
+     * digit year and three digit dayOfYear (yyyy-DDD).
+     */
+    private static final CompoundDateTimeFormatter ORDINAL_DATE =
+        new CompoundDateTimeFormatter(ORDINAL_DATE_PRINTER, ORDINAL_DATE_FORMATTER);
+
+    private static final DateTimeFormatter TIME_NO_MILLIS_FORMATTER = new DateTimeFormatterBuilder()
+        .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral(':')
+        .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral(':')
+        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
+        .toFormatter(Locale.ROOT);
+
+    private static final DateTimeFormatter T_TIME_NO_MILLIS_FORMATTER =
+        new DateTimeFormatterBuilder().appendLiteral("T").append(TIME_NO_MILLIS_FORMATTER).toFormatter(Locale.ROOT);
+
+    private static final DateTimeFormatter TIME_PREFIX = new DateTimeFormatterBuilder()
+        .append(TIME_NO_MILLIS_FORMATTER)
+        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
+        .toFormatter(Locale.ROOT);
+
+    private static final DateTimeFormatter WEEK_DATE_FORMATTER = new DateTimeFormatterBuilder()
+        .appendValue(IsoFields.WEEK_BASED_YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
+        .appendLiteral("-W")
+        .appendValue(IsoFields.WEEK_OF_WEEK_BASED_YEAR, 1, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral('-')
+        .appendValue(DAY_OF_WEEK, 1)
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter for a four digit weekyear. (YYYY)
+     */
+    private static final CompoundDateTimeFormatter WEEK_YEAR = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().appendValue(WeekFields.ISO.weekBasedYear()).toFormatter(Locale.ROOT));
+
+    /*
+     * Returns a formatter for a four digit weekyear. (uuuu)
+     */
+    private static final CompoundDateTimeFormatter YEAR = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().appendValue(ChronoField.YEAR).toFormatter(Locale.ROOT));
+
+    /*
+     * Returns a formatter for parsing the seconds since the epoch
+     */
+    private static final CompoundDateTimeFormatter EPOCH_SECOND = new CompoundDateTimeFormatter(
+        new DateTimeFormatterBuilder().appendValue(ChronoField.INSTANT_SECONDS).toFormatter(Locale.ROOT));
+
+    /*
+     * Returns a formatter for parsing the milliseconds since the epoch
+     */
+    private static final CompoundDateTimeFormatter EPOCH_MILLIS = new CompoundDateTimeFormatter(new DateTimeFormatterBuilder()
+        .appendValue(ChronoField.INSTANT_SECONDS, 1, 19, SignStyle.NEVER)
+        .appendValue(ChronoField.MILLI_OF_SECOND, 3)
+        .toFormatter(Locale.ROOT));
+
+    /*
+     * Returns a formatter that combines a full date and two digit hour of
+     * day. (yyyy-MM-dd'T'HH)
+     */
+    private static final CompoundDateTimeFormatter DATE_HOUR = new CompoundDateTimeFormatter(STRICT_DATE_HOUR.printer,
+        new DateTimeFormatterBuilder()
+            .append(DATE_FORMATTER)
+            .appendLiteral("T")
+            .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
+            .toFormatter(Locale.ROOT));
+
+    /*
+     * Returns a formatter that combines a full date, two digit hour of day,
+     * two digit minute of hour, two digit second of minute, and three digit
+     * fraction of second (yyyy-MM-dd'T'HH:mm:ss.SSS). Parsing will parse up
+     * to 3 fractional second digits.
+     */
+    private static final CompoundDateTimeFormatter DATE_HOUR_MINUTE_SECOND_MILLIS =
+        new CompoundDateTimeFormatter(
+            new DateTimeFormatterBuilder()
+                .append(STRICT_YEAR_MONTH_DAY_FORMATTER)
+                .appendLiteral("T")
+                .append(STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER)
+                .toFormatter(Locale.ROOT),
+            new DateTimeFormatterBuilder()
+                .append(DATE_FORMATTER)
+                .appendLiteral("T")
+                .append(HOUR_MINUTE_SECOND_MILLIS_FORMATTER)
+                .toFormatter(Locale.ROOT));
+
+    private static final CompoundDateTimeFormatter DATE_HOUR_MINUTE_SECOND_FRACTION = DATE_HOUR_MINUTE_SECOND_MILLIS;
+
+    /*
+     * Returns a formatter that combines a full date, two digit hour of day,
+     * and two digit minute of hour. (yyyy-MM-dd'T'HH:mm)
+     */
+    private static final CompoundDateTimeFormatter DATE_HOUR_MINUTE = new CompoundDateTimeFormatter(STRICT_DATE_HOUR_MINUTE.printer,
+        new DateTimeFormatterBuilder()
+            .append(DATE_FORMATTER)
+            .appendLiteral("T")
+            .append(HOUR_MINUTE_FORMATTER)
+            .toFormatter(Locale.ROOT));
+
+    /*
+     * Returns a formatter that combines a full date, two digit hour of day,
+     * two digit minute of hour, and two digit second of
+     * minute. (yyyy-MM-dd'T'HH:mm:ss)
+     */
+    private static final CompoundDateTimeFormatter DATE_HOUR_MINUTE_SECOND = new CompoundDateTimeFormatter(
+        STRICT_DATE_HOUR_MINUTE_SECOND.printer,
+        new DateTimeFormatterBuilder()
+            .append(DATE_FORMATTER)
+            .appendLiteral("T")
+            .append(HOUR_MINUTE_SECOND_FORMATTER)
+            .toFormatter(Locale.ROOT));
+
+    private static final DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder()
+        .append(DATE_FORMATTER)
+        .appendLiteral('T')
+        .append(HOUR_MINUTE_FORMATTER)
+        .optionalStart()
+        .appendLiteral(':')
+        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
+        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
+        .optionalEnd()
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter that combines a full date and time, separated by a 'T'
+     * (yyyy-MM-dd'T'HH:mm:ss.SSSZZ).
+     */
+    private static final CompoundDateTimeFormatter DATE_TIME = new CompoundDateTimeFormatter(
+        STRICT_DATE_TIME.printer,
+        new DateTimeFormatterBuilder().append(DATE_TIME_FORMATTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(DATE_TIME_FORMATTER).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a basic formatter for a full date as four digit weekyear, two
+     * digit week of weekyear, and one digit day of week (YYYY'W'wwe).
+     */
+    private static final CompoundDateTimeFormatter BASIC_WEEK_DATE =
+        new CompoundDateTimeFormatter(STRICT_BASIC_WEEK_DATE.printer, BASIC_WEEK_DATE_FORMATTER);
+
+    /*
+     * Returns a formatter for a full date as four digit year, two digit month
+     * of year, and two digit day of month (yyyy-MM-dd).
+     */
+    private static final CompoundDateTimeFormatter DATE = new CompoundDateTimeFormatter(STRICT_DATE.printer, DATE_FORMATTER);
+
+    // only the formatter, nothing optional here
+    private static final DateTimeFormatter DATE_TIME_NO_MILLIS_PRINTER = new DateTimeFormatterBuilder()
+        .append(STRICT_DATE.printer)
+        .appendLiteral('T')
+        .append(STRICT_HOUR_MINUTE.printer)
+        .appendLiteral(':')
+        .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE)
+        .appendZoneId()
+        .toFormatter(Locale.ROOT);
+
+    private static final DateTimeFormatter DATE_TIME_PREFIX = new DateTimeFormatterBuilder()
+        .append(DATE_FORMATTER)
+        .appendLiteral('T')
+        .append(HOUR_MINUTE_FORMATTER)
+        .optionalStart()
+        .appendLiteral(':')
+        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
+        .optionalEnd()
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter that combines a full date and time without millis, but with a timezone that can be optional
+     * separated by a 'T' (yyyy-MM-dd'T'HH:mm:ssZ).
+     */
+    private static final CompoundDateTimeFormatter DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(DATE_TIME_NO_MILLIS_PRINTER,
+        new DateTimeFormatterBuilder().append(DATE_TIME_PREFIX).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(DATE_TIME_PREFIX).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(DATE_TIME_PREFIX)
+            .optionalStart().appendZoneOrOffsetId().optionalEnd().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(DATE_TIME_PREFIX)
+            .optionalStart().append(TIME_ZONE_FORMATTER_NO_COLON).optionalEnd().toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a formatter for a two digit hour of day, two digit minute of
+     * hour, two digit second of minute, and three digit fraction of
+     * second (HH:mm:ss.SSS).
+     */
+    private static final CompoundDateTimeFormatter HOUR_MINUTE_SECOND_MILLIS =
+        new CompoundDateTimeFormatter(STRICT_HOUR_MINUTE_SECOND_FRACTION.printer, HOUR_MINUTE_SECOND_MILLIS_FORMATTER);
+
+    /*
+     * Returns a formatter for a two digit hour of day and two digit minute of
+     * hour. (HH:mm)
+     */
+    private static final CompoundDateTimeFormatter HOUR_MINUTE =
+        new CompoundDateTimeFormatter(STRICT_HOUR_MINUTE.printer, HOUR_MINUTE_FORMATTER);
+
+    /*
+     * A strict formatter that formats or parses a hour, minute and second, such as '09:43:25'.
+     */
+    private static final CompoundDateTimeFormatter HOUR_MINUTE_SECOND = new CompoundDateTimeFormatter(
+        STRICT_HOUR_MINUTE_SECOND.printer,
+        new DateTimeFormatterBuilder()
+            .append(HOUR_MINUTE_FORMATTER)
+            .appendLiteral(":")
+            .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
+            .toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a formatter for a two digit hour of day. (HH)
+     */
+    private static final CompoundDateTimeFormatter HOUR = new CompoundDateTimeFormatter(
+        STRICT_HOUR.printer,
+        new DateTimeFormatterBuilder().appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE).toFormatter(Locale.ROOT)
+    );
+
+    private static final DateTimeFormatter ORDINAL_DATE_TIME_FORMATTER_BASE = new DateTimeFormatterBuilder()
+        .append(ORDINAL_DATE_FORMATTER)
+        .appendLiteral('T')
+        .append(HOUR_MINUTE_FORMATTER)
+        .optionalStart()
+        .appendLiteral(':')
+        .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
+        .appendFraction(MILLI_OF_SECOND, 1, 3, true)
+        .optionalEnd()
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter for a full ordinal date and time, using a four
+     * digit year and three digit dayOfYear (yyyy-DDD'T'HH:mm:ss.SSSZZ).
+     */
+    private static final CompoundDateTimeFormatter ORDINAL_DATE_TIME = new CompoundDateTimeFormatter(
+        STRICT_ORDINAL_DATE_TIME.printer,
+        new DateTimeFormatterBuilder().append(ORDINAL_DATE_TIME_FORMATTER_BASE)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(ORDINAL_DATE_TIME_FORMATTER_BASE)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    private static final DateTimeFormatter ORDINAL_DATE_TIME_NO_MILLIS_BASE = new DateTimeFormatterBuilder()
+        .append(ORDINAL_DATE_FORMATTER)
+        .appendLiteral('T')
+        .append(HOUR_MINUTE_SECOND_FORMATTER)
+        .toFormatter(Locale.ROOT);
+
+    /*
+     * Returns a formatter for a full ordinal date and time without millis,
+     * using a four digit year and three digit dayOfYear (yyyy-DDD'T'HH:mm:ssZZ).
+     */
+    private static final CompoundDateTimeFormatter ORDINAL_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        STRICT_ORDINAL_DATE_TIME_NO_MILLIS.printer,
+        new DateTimeFormatterBuilder().append(ORDINAL_DATE_TIME_NO_MILLIS_BASE)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(ORDINAL_DATE_TIME_NO_MILLIS_BASE)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a formatter that combines a full weekyear date and time,
+     * separated by a 'T' (xxxx-'W'ww-e'T'HH:mm:ss.SSSZZ).
+     */
+    private static final CompoundDateTimeFormatter WEEK_DATE_TIME = new CompoundDateTimeFormatter(
+        STRICT_WEEK_DATE_TIME.printer,
+        new DateTimeFormatterBuilder().append(WEEK_DATE_FORMATTER).appendLiteral("T").append(TIME_PREFIX)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(WEEK_DATE_FORMATTER).appendLiteral("T").append(TIME_PREFIX)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a formatter that combines a full weekyear date and time,
+     * separated by a 'T' (xxxx-'W'ww-e'T'HH:mm:ssZZ).
+     */
+    private static final CompoundDateTimeFormatter WEEK_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        STRICT_WEEK_DATE_TIME_NO_MILLIS.printer,
+        new DateTimeFormatterBuilder().append(WEEK_DATE_FORMATTER).append(T_TIME_NO_MILLIS_FORMATTER)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(WEEK_DATE_FORMATTER).append(T_TIME_NO_MILLIS_FORMATTER)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a basic formatter that combines a basic weekyear date and time,
+     * separated by a 'T' (xxxx'W'wwe'T'HHmmss.SSSX).
+     */
+    private static final CompoundDateTimeFormatter BASIC_WEEK_DATE_TIME = new CompoundDateTimeFormatter(
+        STRICT_BASIC_WEEK_DATE_TIME.printer,
+        new DateTimeFormatterBuilder().append(BASIC_WEEK_DATE_FORMATTER).append(BASIC_T_TIME_FORMATTER)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(BASIC_WEEK_DATE_FORMATTER).append(BASIC_T_TIME_FORMATTER)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a basic formatter that combines a basic weekyear date and time,
+     * separated by a 'T' (xxxx'W'wwe'T'HHmmssX).
+     */
+    private static final CompoundDateTimeFormatter BASIC_WEEK_DATE_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        STRICT_BASIC_WEEK_DATE_TIME_NO_MILLIS.printer,
+        new DateTimeFormatterBuilder().append(BASIC_WEEK_DATE_FORMATTER).appendLiteral("T").append(BASIC_TIME_NO_MILLIS_BASE)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(BASIC_WEEK_DATE_FORMATTER).appendLiteral("T").append(BASIC_TIME_NO_MILLIS_BASE)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a formatter for a two digit hour of day, two digit minute of
+     * hour, two digit second of minute, three digit fraction of second, and
+     * time zone offset (HH:mm:ss.SSSZZ).
+     */
+    private static final CompoundDateTimeFormatter TIME = new CompoundDateTimeFormatter(
+        STRICT_TIME.printer,
+        new DateTimeFormatterBuilder().append(TIME_PREFIX).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(TIME_PREFIX).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a formatter for a two digit hour of day, two digit minute of
+     * hour, two digit second of minute, andtime zone offset (HH:mm:ssZZ).
+     */
+    private static final CompoundDateTimeFormatter TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        STRICT_TIME_NO_MILLIS.printer,
+        new DateTimeFormatterBuilder().append(TIME_NO_MILLIS_FORMATTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(TIME_NO_MILLIS_FORMATTER).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a formatter for a two digit hour of day, two digit minute of
+     * hour, two digit second of minute, three digit fraction of second, and
+     * time zone offset prefixed by 'T' ('T'HH:mm:ss.SSSZZ).
+     */
+    private static final CompoundDateTimeFormatter T_TIME = new CompoundDateTimeFormatter(
+        STRICT_T_TIME.printer,
+        new DateTimeFormatterBuilder().appendLiteral("T").append(TIME_PREFIX)
+            .appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().appendLiteral("T").append(TIME_PREFIX)
+            .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a formatter for a two digit hour of day, two digit minute of
+     * hour, two digit second of minute, and time zone offset prefixed
+     * by 'T' ('T'HH:mm:ssZZ).
+     */
+    private static final CompoundDateTimeFormatter T_TIME_NO_MILLIS = new CompoundDateTimeFormatter(
+        STRICT_T_TIME_NO_MILLIS.printer,
+        new DateTimeFormatterBuilder().append(T_TIME_NO_MILLIS_FORMATTER).appendZoneOrOffsetId().toFormatter(Locale.ROOT),
+        new DateTimeFormatterBuilder().append(T_TIME_NO_MILLIS_FORMATTER).append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * A strict formatter that formats or parses a year and a month, such as '2011-12'.
+     */
+    private static final CompoundDateTimeFormatter YEAR_MONTH = new CompoundDateTimeFormatter(
+        STRICT_YEAR_MONTH.printer,
+        new DateTimeFormatterBuilder().appendValue(ChronoField.YEAR).appendLiteral("-").appendValue(MONTH_OF_YEAR).toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * A strict date formatter that formats or parses a date without an offset, such as '2011-12-03'.
+     */
+    private static final CompoundDateTimeFormatter YEAR_MONTH_DAY = new CompoundDateTimeFormatter(
+        STRICT_YEAR_MONTH_DAY.printer,
+        new DateTimeFormatterBuilder()
+            .appendValue(ChronoField.YEAR)
+            .appendLiteral("-")
+            .appendValue(MONTH_OF_YEAR)
+            .appendLiteral("-")
+            .appendValue(DAY_OF_MONTH)
+            .toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a formatter for a full date as four digit weekyear, two digit
+     * week of weekyear, and one digit day of week (xxxx-'W'ww-e).
+     */
+    private static final CompoundDateTimeFormatter WEEK_DATE = new CompoundDateTimeFormatter(STRICT_WEEK_DATE.printer, WEEK_DATE_FORMATTER);
+
+    /*
+     * Returns a formatter for a four digit weekyear and two digit week of
+     * weekyear. (xxxx-'W'ww)
+     */
+    private static final CompoundDateTimeFormatter WEEKYEAR_WEEK = new CompoundDateTimeFormatter(STRICT_WEEKYEAR_WEEK.printer,
+        new DateTimeFormatterBuilder()
+            .appendValue(WeekFields.ISO.weekBasedYear())
+            .appendLiteral("-W")
+            .appendValue(WeekFields.ISO.weekOfWeekBasedYear())
+            .toFormatter(Locale.ROOT)
+    );
+
+    /*
+     * Returns a formatter for a four digit weekyear, two digit week of
+     * weekyear, and one digit day of week. (xxxx-'W'ww-e)
+     */
+    private static final CompoundDateTimeFormatter WEEKYEAR_WEEK_DAY = new CompoundDateTimeFormatter(
+        STRICT_WEEKYEAR_WEEK_DAY.printer,
+        new DateTimeFormatterBuilder()
+            .appendValue(WeekFields.ISO.weekBasedYear())
+            .appendLiteral("-W")
+            .appendValue(WeekFields.ISO.weekOfWeekBasedYear())
+            .appendLiteral("-")
+            .appendValue(WeekFields.ISO.dayOfWeek())
+            .toFormatter(Locale.ROOT)
+    );
+
+    /////////////////////////////////////////
+    //
+    // end lenient formatters
+    //
+    /////////////////////////////////////////
+
     public static CompoundDateTimeFormatter forPattern(String input) {
         return forPattern(input, Locale.ROOT);
     }
@@ -791,7 +1216,7 @@ public static CompoundDateTimeFormatter forPattern(String input, Locale locale)
         }
 
         if ("basicDate".equals(input) || "basic_date".equals(input)) {
-            return BASIC_ISO_DATE;
+            return BASIC_DATE;
         } else if ("basicDateTime".equals(input) || "basic_date_time".equals(input)) {
             return BASIC_DATE_TIME;
         } else if ("basicDateTimeNoMillis".equals(input) || "basic_date_time_no_millis".equals(input)) {
@@ -916,7 +1341,7 @@ public static CompoundDateTimeFormatter forPattern(String input, Locale locale)
         } else if ("strictHourMinuteSecondMillis".equals(input) || "strict_hour_minute_second_millis".equals(input)) {
             return STRICT_HOUR_MINUTE_SECOND_MILLIS;
         } else if ("strictOrdinalDate".equals(input) || "strict_ordinal_date".equals(input)) {
-            return ISO_ORDINAL_DATE;
+            return STRICT_ORDINAL_DATE;
         } else if ("strictOrdinalDateTime".equals(input) || "strict_ordinal_date_time".equals(input)) {
             return STRICT_ORDINAL_DATE_TIME;
         } else if ("strictOrdinalDateTimeNoMillis".equals(input) || "strict_ordinal_date_time_no_millis".equals(input)) {
diff --git a/server/src/main/java/org/elasticsearch/common/time/DateMathParser.java b/server/src/main/java/org/elasticsearch/common/time/DateMathParser.java
new file mode 100644
index 0000000000000..39f6dabbdb2e6
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/time/DateMathParser.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.time;
+
+import org.elasticsearch.ElasticsearchParseException;
+
+import java.time.DateTimeException;
+import java.time.DayOfWeek;
+import java.time.Instant;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoField;
+import java.time.temporal.TemporalAccessor;
+import java.time.temporal.TemporalAdjusters;
+import java.time.temporal.TemporalField;
+import java.time.temporal.TemporalQueries;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.LongSupplier;
+
+/**
+ * A parser for date/time formatted text with optional date math.
+ *
+ * The format of the datetime is configurable, and unix timestamps can also be used. Datemath
+ * is appended to a datetime with the following syntax:
+ * <code>||[+-/](\d+)?[yMwdhHms]</code>.
+ */
+public class DateMathParser {
+
+    // base fields which should be used for default parsing, when we round up
+    private static final Map<TemporalField, Long> ROUND_UP_BASE_FIELDS = new HashMap<>(6);
+    {
+        ROUND_UP_BASE_FIELDS.put(ChronoField.MONTH_OF_YEAR, 1L);
+        ROUND_UP_BASE_FIELDS.put(ChronoField.DAY_OF_MONTH, 1L);
+        ROUND_UP_BASE_FIELDS.put(ChronoField.HOUR_OF_DAY, 23L);
+        ROUND_UP_BASE_FIELDS.put(ChronoField.MINUTE_OF_HOUR, 59L);
+        ROUND_UP_BASE_FIELDS.put(ChronoField.SECOND_OF_MINUTE, 59L);
+        ROUND_UP_BASE_FIELDS.put(ChronoField.MILLI_OF_SECOND, 999L);
+    }
+
+    private final CompoundDateTimeFormatter formatter;
+    private final CompoundDateTimeFormatter roundUpFormatter;
+
+    public DateMathParser(CompoundDateTimeFormatter formatter) {
+        Objects.requireNonNull(formatter);
+        this.formatter = formatter;
+        this.roundUpFormatter = formatter.parseDefaulting(ROUND_UP_BASE_FIELDS);
+    }
+
+    public long parse(String text, LongSupplier now) {
+        return parse(text, now, false, null);
+    }
+
+    /**
+     * Parse text, that potentially contains date math into the milliseconds since the epoch
+     *
+     * Examples are
+     *
+     * <code>2014-11-18||-2y</code> substracts two years from the input date
+     * <code>now/m</code>           rounds the current time to minute granularity
+     *
+     * Supported rounding units are
+     * y    year
+     * M    month
+     * w    week (beginning on a monday)
+     * d    day
+     * h/H  hour
+     * m    minute
+     * s    second
+     *
+     *
+     * @param text      the input
+     * @param now       a supplier to retrieve the current date in milliseconds, if needed for additions
+     * @param roundUp   should the result be rounded up
+     * @param timeZone  an optional timezone that should be applied before returning the milliseconds since the epoch
+     * @return          the parsed date in milliseconds since the epoch
+     */
+    public long parse(String text, LongSupplier now, boolean roundUp, ZoneId timeZone) {
+        long time;
+        String mathString;
+        if (text.startsWith("now")) {
+            try {
+                time = now.getAsLong();
+            } catch (Exception e) {
+                throw new ElasticsearchParseException("could not read the current timestamp", e);
+            }
+            mathString = text.substring("now".length());
+        } else {
+            int index = text.indexOf("||");
+            if (index == -1) {
+                return parseDateTime(text, timeZone, roundUp);
+            }
+            time = parseDateTime(text.substring(0, index), timeZone, false);
+            mathString = text.substring(index + 2);
+        }
+
+        return parseMath(mathString, time, roundUp, timeZone);
+    }
+
+    private long parseMath(final String mathString, final long time, final boolean roundUp,
+                           ZoneId timeZone) throws ElasticsearchParseException {
+        if (timeZone == null) {
+            timeZone = ZoneOffset.UTC;
+        }
+        ZonedDateTime dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(time), timeZone);
+        for (int i = 0; i < mathString.length(); ) {
+            char c = mathString.charAt(i++);
+            final boolean round;
+            final int sign;
+            if (c == '/') {
+                round = true;
+                sign = 1;
+            } else {
+                round = false;
+                if (c == '+') {
+                    sign = 1;
+                } else if (c == '-') {
+                    sign = -1;
+                } else {
+                    throw new ElasticsearchParseException("operator not supported for date math [{}]", mathString);
+                }
+            }
+
+            if (i >= mathString.length()) {
+                throw new ElasticsearchParseException("truncated date math [{}]", mathString);
+            }
+
+            final int num;
+            if (!Character.isDigit(mathString.charAt(i))) {
+                num = 1;
+            } else {
+                int numFrom = i;
+                while (i < mathString.length() && Character.isDigit(mathString.charAt(i))) {
+                    i++;
+                }
+                if (i >= mathString.length()) {
+                    throw new ElasticsearchParseException("truncated date math [{}]", mathString);
+                }
+                num = Integer.parseInt(mathString.substring(numFrom, i));
+            }
+            if (round) {
+                if (num != 1) {
+                    throw new ElasticsearchParseException("rounding `/` can only be used on single unit types [{}]", mathString);
+                }
+            }
+            char unit = mathString.charAt(i++);
+            switch (unit) {
+                case 'y':
+                    if (round) {
+                        dateTime = dateTime.withDayOfYear(1).with(LocalTime.MIN);
+                    } else {
+                        dateTime = dateTime.plusYears(sign * num);
+                    }
+                    if (roundUp) {
+                        dateTime = dateTime.plusYears(1);
+                    }
+                    break;
+                case 'M':
+                    if (round) {
+                        dateTime = dateTime.withDayOfMonth(1).with(LocalTime.MIN);
+                    } else {
+                        dateTime = dateTime.plusMonths(sign * num);
+                    }
+                    if (roundUp) {
+                        dateTime = dateTime.plusMonths(1);
+                    }
+                    break;
+                case 'w':
+                    if (round) {
+                        dateTime = dateTime.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)).with(LocalTime.MIN);
+                    } else {
+                        dateTime = dateTime.plusWeeks(sign * num);
+                    }
+                    if (roundUp) {
+                        dateTime = dateTime.plusWeeks(1);
+                    }
+                    break;
+                case 'd':
+                    if (round) {
+                        dateTime = dateTime.with(LocalTime.MIN);
+                    } else {
+                        dateTime = dateTime.plusDays(sign * num);
+                    }
+                    if (roundUp) {
+                        dateTime = dateTime.plusDays(1);
+                    }
+                    break;
+                case 'h':
+                case 'H':
+                    if (round) {
+                        dateTime = dateTime.withMinute(0).withSecond(0).withNano(0);
+                    } else {
+                        dateTime = dateTime.plusHours(sign * num);
+                    }
+                    if (roundUp) {
+                        dateTime = dateTime.plusHours(1);
+                    }
+                    break;
+                case 'm':
+                    if (round) {
+                        dateTime = dateTime.withSecond(0).withNano(0);
+                    } else {
+                        dateTime = dateTime.plusMinutes(sign * num);
+                    }
+                    if (roundUp) {
+                        dateTime = dateTime.plusMinutes(1);
+                    }
+                    break;
+                case 's':
+                    if (round) {
+                        dateTime = dateTime.withNano(0);
+                    } else {
+                        dateTime = dateTime.plusSeconds(sign * num);
+                    }
+                    if (roundUp) {
+                        dateTime = dateTime.plusSeconds(1);
+                    }
+                    break;
+                default:
+                    throw new ElasticsearchParseException("unit [{}] not supported for date math [{}]", unit, mathString);
+            }
+            if (roundUp) {
+                dateTime = dateTime.minus(1, ChronoField.MILLI_OF_SECOND.getBaseUnit());
+            }
+        }
+        return dateTime.toInstant().toEpochMilli();
+    }
+
+    private long parseDateTime(String value, ZoneId timeZone, boolean roundUpIfNoTime) {
+        CompoundDateTimeFormatter formatter = roundUpIfNoTime ? this.roundUpFormatter : this.formatter;
+        try {
+            if (timeZone == null) {
+                return DateFormatters.toZonedDateTime(formatter.parse(value)).toInstant().toEpochMilli();
+            } else {
+                TemporalAccessor accessor = formatter.parse(value);
+                ZoneId zoneId = TemporalQueries.zone().queryFrom(accessor);
+                if (zoneId != null) {
+                    timeZone = zoneId;
+                }
+
+                return DateFormatters.toZonedDateTime(accessor).withZoneSameLocal(timeZone).toInstant().toEpochMilli();
+            }
+        } catch (IllegalArgumentException | DateTimeException e) {
+            throw new ElasticsearchParseException("failed to parse date field [{}]: [{}]", e, value, e.getMessage());
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java b/server/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java
index d7c37febb5dd2..462136a22fe94 100644
--- a/server/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java
+++ b/server/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java
@@ -21,6 +21,7 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -113,7 +114,7 @@ protected synchronized void doStart() {
     }
 
     protected ClusterState createInitialState(DiscoveryNode localNode) {
-        ClusterState.Builder builder = clusterApplier.newClusterStateBuilder();
+        ClusterState.Builder builder = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings));
         return builder.nodes(DiscoveryNodes.builder().add(localNode)
                 .localNodeId(localNode.getId())
                 .masterNodeId(localNode.getId())
diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/server/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index eb9a9f8d4885d..a68557adb9d03 100644
--- a/server/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/server/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -252,7 +252,7 @@ protected void doStart() {
             // set initial state
             assert committedState.get() == null;
             assert localNode != null;
-            ClusterState.Builder builder = clusterApplier.newClusterStateBuilder();
+            ClusterState.Builder builder = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings));
             ClusterState initialState = builder
                 .blocks(ClusterBlocks.builder()
                     .addGlobalBlock(STATE_NOT_RECOVERED_BLOCK)
diff --git a/server/src/main/java/org/elasticsearch/gateway/Gateway.java b/server/src/main/java/org/elasticsearch/gateway/Gateway.java
index ae8f5a85def44..d2261e5d1b421 100644
--- a/server/src/main/java/org/elasticsearch/gateway/Gateway.java
+++ b/server/src/main/java/org/elasticsearch/gateway/Gateway.java
@@ -23,6 +23,7 @@
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -147,7 +148,7 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t
                 metaDataBuilder.transientSettings(),
                 e -> logUnknownSetting("transient", e),
                 (e, ex) -> logInvalidSetting("transient", e, ex)));
-        ClusterState.Builder builder = clusterService.newClusterStateBuilder();
+        ClusterState.Builder builder = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings));
         builder.metaData(metaDataBuilder);
         listener.onSuccess(builder.build());
     }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
index 486515e675562..44cd743bbd428 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.node.Node;
 
 import java.util.Collections;
@@ -254,6 +255,14 @@ public final class IndexSettings {
     public static final Setting<Integer> MAX_REGEX_LENGTH_SETTING = Setting.intSetting("index.max_regex_length",
         1000, 1, Property.Dynamic, Property.IndexScope);
 
+    public static final Setting<String> DEFAULT_PIPELINE =
+       new Setting<>("index.default_pipeline", IngestService.NOOP_PIPELINE_NAME, s -> {
+           if (s == null || s.isEmpty()) {
+               throw new IllegalArgumentException("Value for [index.default_pipeline] must be a non-empty string.");
+           }
+        return s;
+       }, Property.Dynamic, Property.IndexScope);
+
     private final Index index;
     private final Version version;
     private final Logger logger;
@@ -293,6 +302,7 @@ public final class IndexSettings {
     private volatile TimeValue searchIdleAfter;
     private volatile int maxAnalyzedOffset;
     private volatile int maxTermsCount;
+    private volatile String defaultPipeline;
 
     /**
      * The maximum number of refresh listeners allows on this shard.
@@ -408,6 +418,7 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
         this.mergePolicyConfig = new MergePolicyConfig(logger, this);
         this.indexSortConfig = new IndexSortConfig(this);
         searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER);
+        defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE);
 
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio);
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING, mergePolicyConfig::setExpungeDeletesAllowed);
@@ -446,6 +457,7 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
         scopedSettings.addSettingsUpdateConsumer(DEFAULT_FIELD_SETTING, this::setDefaultFields);
         scopedSettings.addSettingsUpdateConsumer(INDEX_SEARCH_IDLE_AFTER, this::setSearchIdleAfter);
         scopedSettings.addSettingsUpdateConsumer(MAX_REGEX_LENGTH_SETTING, this::setMaxRegexLength);
+        scopedSettings.addSettingsUpdateConsumer(DEFAULT_PIPELINE, this::setDefaultPipeline);
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) { this.searchIdleAfter = searchIdleAfter; }
@@ -821,4 +833,12 @@ public boolean isExplicitRefresh() {
      * Returns the time that an index shard becomes search idle unless it's accessed in between
      */
     public TimeValue getSearchIdleAfter() { return searchIdleAfter; }
+
+    public String getDefaultPipeline() {
+        return defaultPipeline;
+    }
+
+    public void setDefaultPipeline(String defaultPipeline) {
+        this.defaultPipeline = defaultPipeline;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/ESSolrSynonymParser.java b/server/src/main/java/org/elasticsearch/index/analysis/ESSolrSynonymParser.java
index bcc249f8a8a51..006973dd9b6bc 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/ESSolrSynonymParser.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/ESSolrSynonymParser.java
@@ -20,19 +20,18 @@
 package org.elasticsearch.index.analysis;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.synonym.SolrSynonymParser;
 import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.common.logging.Loggers;
 
 import java.io.IOException;
 
 public class ESSolrSynonymParser extends SolrSynonymParser {
+    private static final Logger logger = LogManager.getLogger(ESSolrSynonymParser.class);
 
     private final boolean lenient;
-    private static final Logger logger =
-        Loggers.getLogger(ESSolrSynonymParser.class, "ESSolrSynonymParser");
 
     public ESSolrSynonymParser(boolean dedup, boolean expand, boolean lenient, Analyzer analyzer) {
         super(dedup, expand, analyzer);
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/ESWordnetSynonymParser.java b/server/src/main/java/org/elasticsearch/index/analysis/ESWordnetSynonymParser.java
index 3764820c4343d..ebcd84e39d7a2 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/ESWordnetSynonymParser.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/ESWordnetSynonymParser.java
@@ -20,19 +20,18 @@
 package org.elasticsearch.index.analysis;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.synonym.WordnetSynonymParser;
 import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.common.logging.Loggers;
 
 import java.io.IOException;
 
 public class ESWordnetSynonymParser extends WordnetSynonymParser {
+    private static final Logger logger = LogManager.getLogger(ESWordnetSynonymParser.class);
 
     private final boolean lenient;
-    private static final Logger logger =
-        Loggers.getLogger(ESSolrSynonymParser.class, "ESWordnetSynonymParser");
 
     public ESWordnetSynonymParser(boolean dedup, boolean expand, boolean lenient, Analyzer analyzer) {
         super(dedup, expand, analyzer);
diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index b7c938b469fa6..31da7afc51a10 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -304,6 +304,7 @@ public abstract static class Result {
         private final Operation.TYPE operationType;
         private final Result.Type resultType;
         private final long version;
+        private final long term;
         private final long seqNo;
         private final Exception failure;
         private final SetOnce<Boolean> freeze = new SetOnce<>();
@@ -311,19 +312,21 @@ public abstract static class Result {
         private Translog.Location translogLocation;
         private long took;
 
-        protected Result(Operation.TYPE operationType, Exception failure, long version, long seqNo) {
+        protected Result(Operation.TYPE operationType, Exception failure, long version, long term, long seqNo) {
             this.operationType = operationType;
             this.failure = Objects.requireNonNull(failure);
             this.version = version;
+            this.term = term;
             this.seqNo = seqNo;
             this.requiredMappingUpdate = null;
             this.resultType = Type.FAILURE;
         }
 
-        protected Result(Operation.TYPE operationType, long version, long seqNo) {
+        protected Result(Operation.TYPE operationType, long version, long term, long seqNo) {
             this.operationType = operationType;
             this.version = version;
             this.seqNo = seqNo;
+            this.term = term;
             this.failure = null;
             this.requiredMappingUpdate = null;
             this.resultType = Type.SUCCESS;
@@ -333,6 +336,7 @@ protected Result(Operation.TYPE operationType, Mapping requiredMappingUpdate) {
             this.operationType = operationType;
             this.version = Versions.NOT_FOUND;
             this.seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
+            this.term = 0L;
             this.failure = null;
             this.requiredMappingUpdate = requiredMappingUpdate;
             this.resultType = Type.MAPPING_UPDATE_REQUIRED;
@@ -357,6 +361,10 @@ public long getSeqNo() {
             return seqNo;
         }
 
+        public long getTerm() {
+            return term;
+        }
+
         /**
          * If the operation was aborted due to missing mappings, this method will return the mappings
          * that are required to complete the operation.
@@ -415,20 +423,20 @@ public static class IndexResult extends Result {
 
         private final boolean created;
 
-        public IndexResult(long version, long seqNo, boolean created) {
-            super(Operation.TYPE.INDEX, version, seqNo);
+        public IndexResult(long version, long term, long seqNo, boolean created) {
+            super(Operation.TYPE.INDEX, version, term, seqNo);
             this.created = created;
         }
 
         /**
          * use in case of the index operation failed before getting to internal engine
          **/
-        public IndexResult(Exception failure, long version) {
-            this(failure, version, SequenceNumbers.UNASSIGNED_SEQ_NO);
+        public IndexResult(Exception failure, long version, long term) {
+            this(failure, version, term, SequenceNumbers.UNASSIGNED_SEQ_NO);
         }
 
-        public IndexResult(Exception failure, long version, long seqNo) {
-            super(Operation.TYPE.INDEX, failure, version, seqNo);
+        public IndexResult(Exception failure, long version, long term, long seqNo) {
+            super(Operation.TYPE.INDEX, failure, version, term, seqNo);
             this.created = false;
         }
 
@@ -447,20 +455,20 @@ public static class DeleteResult extends Result {
 
         private final boolean found;
 
-        public DeleteResult(long version, long seqNo, boolean found) {
-            super(Operation.TYPE.DELETE, version, seqNo);
+        public DeleteResult(long version, long term, long seqNo, boolean found) {
+            super(Operation.TYPE.DELETE, version, term, seqNo);
             this.found = found;
         }
 
         /**
          * use in case of the delete operation failed before getting to internal engine
          **/
-        public DeleteResult(Exception failure, long version) {
-            this(failure, version, SequenceNumbers.UNASSIGNED_SEQ_NO, false);
+        public DeleteResult(Exception failure, long version, long term) {
+            this(failure, version, term, SequenceNumbers.UNASSIGNED_SEQ_NO, false);
         }
 
-        public DeleteResult(Exception failure, long version, long seqNo, boolean found) {
-            super(Operation.TYPE.DELETE, failure, version, seqNo);
+        public DeleteResult(Exception failure, long version, long term, long seqNo, boolean found) {
+            super(Operation.TYPE.DELETE, failure, version, term, seqNo);
             this.found = found;
         }
 
@@ -477,12 +485,12 @@ public boolean isFound() {
 
     public static class NoOpResult extends Result {
 
-        NoOpResult(long seqNo) {
-            super(Operation.TYPE.NO_OP, 0, seqNo);
+        NoOpResult(long term, long seqNo) {
+            super(Operation.TYPE.NO_OP, term, 0, seqNo);
         }
 
-        NoOpResult(long seqNo, Exception failure) {
-            super(Operation.TYPE.NO_OP, failure, 0, seqNo);
+        NoOpResult(long term, long seqNo, Exception failure) {
+            super(Operation.TYPE.NO_OP, failure, term, 0, seqNo);
         }
 
     }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index bdcfb2fc7313f..a30127a24ae21 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -736,6 +736,10 @@ protected long doGenerateSeqNoForOperation(final Operation operation) {
         return localCheckpointTracker.generateSeqNo();
     }
 
+    private long getPrimaryTerm() {
+        return engineConfig.getPrimaryTermSupplier().getAsLong();
+    }
+
     @Override
     public IndexResult index(Index index) throws IOException {
         assert Objects.equals(index.uid().field(), IdFieldMapper.NAME) : index.uid().field();
@@ -788,7 +792,7 @@ public IndexResult index(Index index) throws IOException {
                     indexResult = indexIntoLucene(index, plan);
                 } else {
                     indexResult = new IndexResult(
-                            plan.versionForIndexing, plan.seqNoForIndexing, plan.currentNotFoundOrDeleted);
+                            plan.versionForIndexing, getPrimaryTerm(), plan.seqNoForIndexing, plan.currentNotFoundOrDeleted);
                 }
                 if (index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
                     final Translog.Location location;
@@ -900,7 +904,7 @@ private IndexingStrategy planIndexingAsPrimary(Index index) throws IOException {
                 currentVersion, index.version(), currentNotFoundOrDeleted)) {
                 final VersionConflictEngineException e =
                         new VersionConflictEngineException(shardId, index, currentVersion, currentNotFoundOrDeleted);
-                plan = IndexingStrategy.skipDueToVersionConflict(e, currentNotFoundOrDeleted, currentVersion);
+                plan = IndexingStrategy.skipDueToVersionConflict(e, currentNotFoundOrDeleted, currentVersion, getPrimaryTerm());
             } else {
                 plan = IndexingStrategy.processNormally(currentNotFoundOrDeleted,
                     generateSeqNoForOperation(index),
@@ -930,7 +934,7 @@ private IndexResult indexIntoLucene(Index index, IndexingStrategy plan)
                 assert assertDocDoesNotExist(index, canOptimizeAddDocument(index) == false);
                 addDocs(index.docs(), indexWriter);
             }
-            return new IndexResult(plan.versionForIndexing, plan.seqNoForIndexing, plan.currentNotFoundOrDeleted);
+            return new IndexResult(plan.versionForIndexing, getPrimaryTerm(), plan.seqNoForIndexing, plan.currentNotFoundOrDeleted);
         } catch (Exception ex) {
             if (indexWriter.getTragicException() == null) {
                 /* There is no tragic event recorded so this must be a document failure.
@@ -946,7 +950,7 @@ private IndexResult indexIntoLucene(Index index, IndexingStrategy plan)
                  * we return a `MATCH_ANY` version to indicate no document was index. The value is
                  * not used anyway
                  */
-                return new IndexResult(ex, Versions.MATCH_ANY, plan.seqNoForIndexing);
+                return new IndexResult(ex, Versions.MATCH_ANY, getPrimaryTerm(), plan.seqNoForIndexing);
             } else {
                 throw ex;
             }
@@ -1019,8 +1023,8 @@ static IndexingStrategy optimizedAppendOnly(long seqNoForIndexing) {
         }
 
         static IndexingStrategy skipDueToVersionConflict(
-                VersionConflictEngineException e, boolean currentNotFoundOrDeleted, long currentVersion) {
-            final IndexResult result = new IndexResult(e, currentVersion);
+                VersionConflictEngineException e, boolean currentNotFoundOrDeleted, long currentVersion, long term) {
+            final IndexResult result = new IndexResult(e, currentVersion, term);
             return new IndexingStrategy(
                     currentNotFoundOrDeleted, false, false, SequenceNumbers.UNASSIGNED_SEQ_NO, Versions.NOT_FOUND, result);
         }
@@ -1097,7 +1101,7 @@ public DeleteResult delete(Delete delete) throws IOException {
                 deleteResult = deleteInLucene(delete, plan);
             } else {
                 deleteResult = new DeleteResult(
-                        plan.versionOfDeletion, plan.seqNoOfDeletion, plan.currentlyDeleted == false);
+                        plan.versionOfDeletion, getPrimaryTerm(), plan.seqNoOfDeletion, plan.currentlyDeleted == false);
             }
             if (delete.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
                 final Translog.Location location;
@@ -1178,7 +1182,7 @@ private DeletionStrategy planDeletionAsPrimary(Delete delete) throws IOException
         final DeletionStrategy plan;
         if (delete.versionType().isVersionConflictForWrites(currentVersion, delete.version(), currentlyDeleted)) {
             final VersionConflictEngineException e = new VersionConflictEngineException(shardId, delete, currentVersion, currentlyDeleted);
-            plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, currentlyDeleted);
+            plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, getPrimaryTerm(), currentlyDeleted);
         } else {
             plan = DeletionStrategy.processNormally(
                     currentlyDeleted,
@@ -1201,12 +1205,12 @@ private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan)
                 new DeleteVersionValue(plan.versionOfDeletion, plan.seqNoOfDeletion, delete.primaryTerm(),
                     engineConfig.getThreadPool().relativeTimeInMillis()));
             return new DeleteResult(
-                plan.versionOfDeletion, plan.seqNoOfDeletion, plan.currentlyDeleted == false);
+                plan.versionOfDeletion, getPrimaryTerm(), plan.seqNoOfDeletion, plan.currentlyDeleted == false);
         } catch (Exception ex) {
             if (indexWriter.getTragicException() == null) {
                 // there is no tragic event and such it must be a document level failure
                 return new DeleteResult(
-                        ex, plan.versionOfDeletion, plan.seqNoOfDeletion, plan.currentlyDeleted == false);
+                        ex, plan.versionOfDeletion, getPrimaryTerm(), plan.seqNoOfDeletion, plan.currentlyDeleted == false);
             } else {
                 throw ex;
             }
@@ -1237,9 +1241,9 @@ private DeletionStrategy(boolean deleteFromLucene, boolean currentlyDeleted,
         }
 
         static DeletionStrategy skipDueToVersionConflict(
-                VersionConflictEngineException e, long currentVersion, boolean currentlyDeleted) {
+                VersionConflictEngineException e, long currentVersion, long term, boolean currentlyDeleted) {
             final long unassignedSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
-            final DeleteResult deleteResult = new DeleteResult(e, currentVersion, unassignedSeqNo, currentlyDeleted == false);
+            final DeleteResult deleteResult = new DeleteResult(e, currentVersion, term, unassignedSeqNo, currentlyDeleted == false);
             return new DeletionStrategy(false, currentlyDeleted, unassignedSeqNo, Versions.NOT_FOUND, deleteResult);
         }
 
@@ -1268,7 +1272,7 @@ public NoOpResult noOp(final NoOp noOp) {
         try (ReleasableLock ignored = readLock.acquire()) {
             noOpResult = innerNoOp(noOp);
         } catch (final Exception e) {
-            noOpResult = new NoOpResult(noOp.seqNo(), e);
+            noOpResult = new NoOpResult(getPrimaryTerm(), noOp.seqNo(), e);
         }
         return noOpResult;
     }
@@ -1278,7 +1282,7 @@ private NoOpResult innerNoOp(final NoOp noOp) throws IOException {
         assert noOp.seqNo() > SequenceNumbers.NO_OPS_PERFORMED;
         final long seqNo = noOp.seqNo();
         try {
-            final NoOpResult noOpResult = new NoOpResult(noOp.seqNo());
+            final NoOpResult noOpResult = new NoOpResult(getPrimaryTerm(), noOp.seqNo());
             if (noOp.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
                 final Translog.Location location = translog.add(new Translog.NoOp(noOp.seqNo(), noOp.primaryTerm(), noOp.reason()));
                 noOpResult.setTranslogLocation(location);
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
index fedad6e134b46..35284cb655d98 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.fielddata;
 
-
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
@@ -29,18 +28,23 @@
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.MutableDateTime;
-import org.joda.time.ReadableDateTime;
 
 import java.io.IOException;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
 import java.util.AbstractList;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
+import java.util.function.Consumer;
 import java.util.function.UnaryOperator;
 
+import static org.elasticsearch.common.Booleans.parseBoolean;
 
 /**
  * Script level doc values, the assumption is that any implementation will
@@ -52,6 +56,7 @@
  * values form multiple documents.
  */
 public abstract class ScriptDocValues<T> extends AbstractList<T> {
+
     /**
      * Set the current doc ID.
      */
@@ -142,31 +147,55 @@ public int size() {
         }
     }
 
-    public static final class Dates extends ScriptDocValues<ReadableDateTime> {
-        protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Dates.class));
+    public static final class Dates extends ScriptDocValues<Object> {
 
-        private static final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC);
+        /** Whether scripts should expose dates as java time objects instead of joda time. */
+        private static final boolean USE_JAVA_TIME = parseBoolean(System.getProperty("es.scripting.use_java_time"), false);
+
+        private static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Dates.class));
 
         private final SortedNumericDocValues in;
+
+        /**
+         * Method call to add deprecation message. Normally this is
+         * {@link #deprecationLogger} but tests override.
+         */
+        private final Consumer<String> deprecationCallback;
+
         /**
-         * Values wrapped in {@link MutableDateTime}. Null by default an allocated on first usage so we allocate a reasonably size. We keep
-         * this array so we don't have allocate new {@link MutableDateTime}s on every usage. Instead we reuse them for every document.
+         * Whether java time or joda time should be used. This is normally {@link #USE_JAVA_TIME} but tests override it.
          */
-        private MutableDateTime[] dates;
+        private final boolean useJavaTime;
+
+        /**
+         * Values wrapped in a date time object. The concrete type depends on the system property {@code es.scripting.use_java_time}.
+         * When that system property is {@code false}, the date time objects are of type {@link MutableDateTime}. When the system
+         * property is {@code true}, the date time objects are of type {@link java.time.ZonedDateTime}.
+         */
+        private Object[] dates;
         private int count;
 
         /**
          * Standard constructor.
          */
         public Dates(SortedNumericDocValues in) {
+            this(in, message -> deprecationLogger.deprecatedAndMaybeLog("scripting_joda_time_deprecation", message), USE_JAVA_TIME);
+        }
+
+        /**
+         * Constructor for testing with a deprecation callback.
+         */
+        Dates(SortedNumericDocValues in, Consumer<String> deprecationCallback, boolean useJavaTime) {
             this.in = in;
+            this.deprecationCallback = deprecationCallback;
+            this.useJavaTime = useJavaTime;
         }
 
         /**
          * Fetch the first field value or 0 millis after epoch if there are no
          * in.
          */
-        public ReadableDateTime getValue() {
+        public Object getValue() {
             if (count == 0) {
                 throw new IllegalStateException("A document doesn't have a value for a field! " +
                     "Use doc[<field>].size()==0 to check if a document is missing a field!");
@@ -175,7 +204,7 @@ public ReadableDateTime getValue() {
         }
 
         @Override
-        public ReadableDateTime get(int index) {
+        public Object get(int index) {
             if (index >= count) {
                 throw new IndexOutOfBoundsException(
                         "attempted to fetch the [" + index + "] date when there are only ["
@@ -206,31 +235,42 @@ void refreshArray() throws IOException {
             if (count == 0) {
                 return;
             }
-            if (dates == null) {
-                // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size.
-                dates = new MutableDateTime[count];
-                for (int i = 0; i < dates.length; i++) {
-                    dates[i] = new MutableDateTime(in.nextValue(), DateTimeZone.UTC);
+            if (useJavaTime) {
+                if (dates == null || count > dates.length) {
+                    // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size.
+                    dates = new ZonedDateTime[count];
                 }
-                return;
-            }
-            if (count > dates.length) {
-                // Happens when we move to a new document and it has more dates than any documents before it.
-                MutableDateTime[] backup = dates;
-                dates = new MutableDateTime[count];
-                System.arraycopy(backup, 0, dates, 0, backup.length);
-                for (int i = 0; i < backup.length; i++) {
-                    dates[i].setMillis(in.nextValue());
+                for (int i = 0; i < count; ++i) {
+                    dates[i] = ZonedDateTime.ofInstant(Instant.ofEpochMilli(in.nextValue()), ZoneOffset.UTC);
                 }
-                for (int i = backup.length; i < dates.length; i++) {
+            } else {
+                deprecated("The joda time api for doc values is deprecated. Use -Des.scripting.use_java_time=true" +
+                           " to use the java time api for date field doc values");
+                if (dates == null || count > dates.length) {
+                    // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size.
+                    dates = new MutableDateTime[count];
+                }
+                for (int i = 0; i < count; i++) {
                     dates[i] = new MutableDateTime(in.nextValue(), DateTimeZone.UTC);
                 }
-                return;
-            }
-            for (int i = 0; i < count; i++) {
-                dates[i] = new MutableDateTime(in.nextValue(), DateTimeZone.UTC);
             }
         }
+
+        /**
+         * Log a deprecation log, with the server's permissions, not the permissions of the
+         * script calling this method. We need to do this to prevent errors when rolling
+         * the log file.
+         */
+        private void deprecated(String message) {
+            // Intentionally not calling SpecialPermission.check because this is supposed to be called by scripts
+            AccessController.doPrivileged(new PrivilegedAction<Void>() {
+                @Override
+                public Void run() {
+                    deprecationCallback.accept(message);
+                    return null;
+                }
+            });
+        }
     }
 
     public static final class Doubles extends ScriptDocValues<Double> {
diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
index a3f83609037e1..ba5c4cd929fd2 100644
--- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.get;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.document.DocumentField;
@@ -30,6 +31,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.IgnoredFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.search.lookup.SourceLookup;
 
@@ -225,10 +227,13 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
                 }
             }
         }
-
         for (DocumentField field : metaFields) {
-            Object value = field.getValue();
-            builder.field(field.getName(), value);
+            // TODO: can we avoid having an exception here?
+            if (field.getName().equals(IgnoredFieldMapper.NAME)) {
+                builder.field(field.getName(), field.getValues());
+            } else {
+                builder.field(field.getName(), field.<Object>getValue());
+            }
         }
 
         builder.field(FOUND, exists);
@@ -316,7 +321,11 @@ public static GetResult fromXContentEmbedded(XContentParser parser, String index
                     parser.skipChildren(); // skip potential inner objects for forward compatibility
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                parser.skipChildren(); // skip potential inner arrays for forward compatibility
+                if (IgnoredFieldMapper.NAME.equals(currentFieldName)) {
+                    fields.put(currentFieldName, new DocumentField(currentFieldName, parser.list()));
+                } else {
+                    parser.skipChildren(); // skip potential inner arrays for forward compatibility
+                }
             }
         }
         return new GetResult(index, type, id, version, found, source, fields);
@@ -400,7 +409,12 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(index, type, id, version, exists, fields, sourceAsMap());
+        return Objects.hash(version, exists, index, type, id, fields, sourceAsMap());
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
     }
 }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index cbb008c9d0051..2e949f027d132 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -276,7 +276,8 @@ public Mapper parse(ParseContext context) throws IOException {
                 context.doc().add(field);
             }
         } catch (Exception e) {
-            throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
+            throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(),
+                    fieldType().typeName());
         }
         multiFields.parse(this, context);
         return null;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
index 318d9cfc6fa7e..fb9e16cbe13d0 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
@@ -510,7 +510,8 @@ public Mapper parse(ParseContext context) throws IOException {
             indexShape(context, shape);
         } catch (Exception e) {
             if (ignoreMalformed.value() == false) {
-                throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
+                throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(),
+                        fieldType().typeName());
             }
             context.addIgnoredField(fieldType.name());
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
index 4061303416b7c..bb048ab9afac4 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
@@ -126,7 +126,7 @@ public Query existsQuery(QueryShardContext context) {
          */
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
-            if (isSameIndex(value, context.getFullyQualifiedIndexName())) {
+            if (isSameIndex(value, context.getFullyQualifiedIndex().getName())) {
                 return Queries.newMatchAllQuery();
             } else {
                 return Queries.newMatchNoDocsQuery("Index didn't match. Index queried: " + context.index().getName() + " vs. " + value);
@@ -139,14 +139,14 @@ public Query termsQuery(List values, QueryShardContext context) {
                 return super.termsQuery(values, context);
             }
             for (Object value : values) {
-                if (isSameIndex(value, context.getFullyQualifiedIndexName())) {
+                if (isSameIndex(value, context.getFullyQualifiedIndex().getName())) {
                     // No need to OR these clauses - we can only logically be
                     // running in the context of just one of these index names.
                     return Queries.newMatchAllQuery();
                 }
             }
             // None of the listed index names are this one
-            return Queries.newMatchNoDocsQuery("Index didn't match. Index queried: " + context.getFullyQualifiedIndexName()
+            return Queries.newMatchNoDocsQuery("Index didn't match. Index queried: " + context.getFullyQualifiedIndex().getName()
                 + " vs. " + values);
         }
 
@@ -189,5 +189,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     protected void doMerge(Mapper mergeWith) {
         // nothing to do
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
index 598a6f38a2ef8..ac19298ae322d 100644
--- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
+++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
@@ -83,7 +83,7 @@ public class QueryShardContext extends QueryRewriteContext {
     private String[] types = Strings.EMPTY_ARRAY;
     private boolean cachable = true;
     private final SetOnce<Boolean> frozen = new SetOnce<>();
-    private final String fullyQualifiedIndexName;
+    private final Index fullyQualifiedIndex;
 
     public void setTypes(String... types) {
         this.types = types;
@@ -116,7 +116,8 @@ public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterC
         this.indexSettings = indexSettings;
         this.reader = reader;
         this.clusterAlias = clusterAlias;
-        this.fullyQualifiedIndexName = RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName());
+        this.fullyQualifiedIndex = new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()),
+            indexSettings.getIndex().getUUID());
     }
 
     public QueryShardContext(QueryShardContext source) {
@@ -163,7 +164,7 @@ public BitSetProducer bitsetFilter(Query filter) {
     }
 
     public <IFD extends IndexFieldData<?>> IFD getForField(MappedFieldType fieldType) {
-        return (IFD) indexFieldDataService.apply(fieldType, fullyQualifiedIndexName);
+        return (IFD) indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName());
     }
 
     public void addNamedQuery(String name, Query query) {
@@ -275,7 +276,7 @@ public Collection<String> queryTypes() {
     public SearchLookup lookup() {
         if (lookup == null) {
             lookup = new SearchLookup(getMapperService(),
-                mappedFieldType -> indexFieldDataService.apply(mappedFieldType, fullyQualifiedIndexName), types);
+                mappedFieldType -> indexFieldDataService.apply(mappedFieldType, fullyQualifiedIndex.getName()), types);
         }
         return lookup;
     }
@@ -426,9 +427,9 @@ public IndexReader getIndexReader() {
     }
 
     /**
-     * Returns the fully qualified index name including a remote cluster alias if applicable
+     * Returns the fully qualified index including a remote cluster alias if applicable, and the index uuid
      */
-    public String getFullyQualifiedIndexName() {
-        return fullyQualifiedIndexName;
+    public Index getFullyQualifiedIndex() {
+        return fullyQualifiedIndex;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardException.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardException.java
index 9b6ce3a6e4b51..843ff931d4fc2 100644
--- a/server/src/main/java/org/elasticsearch/index/query/QueryShardException.java
+++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardException.java
@@ -37,16 +37,15 @@ public QueryShardException(QueryShardContext context, String msg, Object... args
     }
 
     public QueryShardException(QueryShardContext context, String msg, Throwable cause, Object... args) {
-        super(msg, cause, args);
-        setIndex(context.index());
+        this(context.getFullyQualifiedIndex(), msg, cause, args);
     }
 
     /**
      * This constructor is provided for use in unit tests where a
      * {@link QueryShardContext} may not be available
      */
-    public QueryShardException(Index index, String msg, Throwable cause) {
-        super(msg, cause);
+    public QueryShardException(Index index, String msg, Throwable cause, Object... args) {
+        super(msg, cause, args);
         setIndex(index);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
index aa8543175d9f0..f848e8722c719 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
@@ -23,8 +23,11 @@
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.tasks.TaskId;
 
+import java.io.IOException;
+
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 /**
@@ -53,6 +56,10 @@ public DeleteByQueryRequest(SearchRequest search) {
         this(search, true);
     }
 
+    public DeleteByQueryRequest(StreamInput in) throws IOException {
+        super.readFrom(in);
+    }
+
     private DeleteByQueryRequest(SearchRequest search, boolean setDefaults) {
         super(search, setDefaults);
         // Delete-By-Query does not require the source
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
index 276c455915323..e45d039edaeae 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
@@ -59,6 +59,13 @@ private ReindexRequest(SearchRequest search, IndexRequest destination, boolean s
         this.destination = destination;
     }
 
+    public ReindexRequest(StreamInput in) throws IOException {
+        super.readFrom(in);
+        destination = new IndexRequest();
+        destination.readFrom(in);
+        remoteInfo = in.readOptionalWriteable(RemoteInfo::new);
+    }
+
     @Override
     protected ReindexRequest self() {
         return this;
@@ -135,10 +142,7 @@ public ReindexRequest forSlice(TaskId slicingTask, SearchRequest slice, int tota
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        destination = new IndexRequest();
-        destination.readFrom(in);
-        remoteInfo = in.readOptionalWriteable(RemoteInfo::new);
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
index 2be0e46dbcedf..eb4fd59a7bc5f 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
@@ -47,6 +47,11 @@ public UpdateByQueryRequest(SearchRequest search) {
         this(search, true);
     }
 
+    public UpdateByQueryRequest(StreamInput in) throws IOException {
+        super.readFrom(in);
+        pipeline = in.readOptionalString();
+    }
+
     private UpdateByQueryRequest(SearchRequest search, boolean setDefaults) {
         super(search, setDefaults);
     }
@@ -108,8 +113,7 @@ public IndicesOptions indicesOptions() {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        pipeline = in.readOptionalString();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
index 6548aad767006..b406621e978da 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
@@ -39,6 +39,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.OptionalLong;
 import java.util.Set;
 import java.util.function.Function;
@@ -85,6 +86,7 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L
      *   computation from that point on.
      */
     volatile boolean primaryMode;
+
     /**
      * Boolean flag that indicates if a relocation handoff is in progress. A handoff is started by calling {@link #startRelocationHandoff}
      * and is finished by either calling {@link #completeRelocationHandoff} or {@link #abortRelocationHandoff}, depending on whether the
@@ -102,6 +104,11 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L
      */
     boolean handoffInProgress;
 
+    /**
+     * Boolean flag that indicates whether a relocation handoff completed (see {@link #completeRelocationHandoff}).
+     */
+    volatile boolean relocated;
+
     /**
      * The global checkpoint tracker relies on the property that cluster state updates are applied in-order. After transferring a primary
      * context from the primary relocation source to the target and initializing the target, it is possible for the target to apply a
@@ -121,6 +128,13 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L
      */
     final Map<String, CheckpointState> checkpoints;
 
+    /**
+     * A callback invoked when the global checkpoint is updated. For primary mode this occurs if the computed global checkpoint advances on
+     * the basis of state changes tracked here. For non-primary mode this occurs if the local knowledge of the global checkpoint advances
+     * due to an update from the primary.
+     */
+    private final LongConsumer onGlobalCheckpointUpdated;
+
     /**
      * This set contains allocation IDs for which there is a thread actively waiting for the local checkpoint to advance to at least the
      * current global checkpoint.
@@ -260,6 +274,13 @@ public boolean isPrimaryMode() {
         return primaryMode;
     }
 
+    /**
+     * Returns whether the replication tracker has relocated away to another shard copy.
+     */
+    public boolean isRelocated() {
+        return relocated;
+    }
+
     /**
      * Class invariant that should hold before and after every invocation of public methods on this class. As Java lacks implication
      * as a logical operator, many of the invariants are written under the form (!A || B), they should be read as (A implies B) however.
@@ -287,6 +308,9 @@ private boolean invariant() {
         // relocation handoff can only occur in primary mode
         assert !handoffInProgress || primaryMode;
 
+        // a relocated copy is not in primary mode
+        assert !relocated || !primaryMode;
+
         // the current shard is marked as in-sync when the global checkpoint tracker operates in primary mode
         assert !primaryMode || checkpoints.get(shardAllocationId).inSync;
 
@@ -375,7 +399,8 @@ public ReplicationTracker(
             final ShardId shardId,
             final String allocationId,
             final IndexSettings indexSettings,
-            final long globalCheckpoint) {
+            final long globalCheckpoint,
+            final LongConsumer onGlobalCheckpointUpdated) {
         super(shardId, indexSettings);
         assert globalCheckpoint >= SequenceNumbers.UNASSIGNED_SEQ_NO : "illegal initial global checkpoint: " + globalCheckpoint;
         this.shardAllocationId = allocationId;
@@ -384,6 +409,7 @@ public ReplicationTracker(
         this.appliedClusterStateVersion = -1L;
         this.checkpoints = new HashMap<>(1 + indexSettings.getNumberOfReplicas());
         checkpoints.put(allocationId, new CheckpointState(SequenceNumbers.UNASSIGNED_SEQ_NO, globalCheckpoint, false, false));
+        this.onGlobalCheckpointUpdated = Objects.requireNonNull(onGlobalCheckpointUpdated);
         this.pendingInSync = new HashSet<>();
         this.routingTable = null;
         this.replicationGroup = null;
@@ -440,7 +466,10 @@ public synchronized void updateGlobalCheckpointOnReplica(final long globalCheckp
         updateGlobalCheckpoint(
                 shardAllocationId,
                 globalCheckpoint,
-                current -> logger.trace("updating global checkpoint from [{}] to [{}] due to [{}]", current, globalCheckpoint, reason));
+                current -> {
+                    logger.trace("updated global checkpoint from [{}] to [{}] due to [{}]", current, globalCheckpoint, reason);
+                    onGlobalCheckpointUpdated.accept(globalCheckpoint);
+                });
         assert invariant();
     }
 
@@ -458,7 +487,7 @@ public synchronized void updateGlobalCheckpointForShard(final String allocationI
                 allocationId,
                 globalCheckpoint,
                 current -> logger.trace(
-                        "updating local knowledge for [{}] on the primary of the global checkpoint from [{}] to [{}]",
+                        "updated local knowledge for [{}] on the primary of the global checkpoint from [{}] to [{}]",
                         allocationId,
                         current,
                         globalCheckpoint));
@@ -469,8 +498,8 @@ private void updateGlobalCheckpoint(final String allocationId, final long global
         final CheckpointState cps = checkpoints.get(allocationId);
         assert !this.shardAllocationId.equals(allocationId) || cps != null;
         if (cps != null && globalCheckpoint > cps.globalCheckpoint) {
-            ifUpdated.accept(cps.globalCheckpoint);
             cps.globalCheckpoint = globalCheckpoint;
+            ifUpdated.accept(cps.globalCheckpoint);
         }
     }
 
@@ -721,8 +750,9 @@ private synchronized void updateGlobalCheckpointOnPrimary() {
         assert computedGlobalCheckpoint >= globalCheckpoint : "new global checkpoint [" + computedGlobalCheckpoint +
             "] is lower than previous one [" + globalCheckpoint + "]";
         if (globalCheckpoint != computedGlobalCheckpoint) {
-            logger.trace("global checkpoint updated to [{}]", computedGlobalCheckpoint);
             cps.globalCheckpoint = computedGlobalCheckpoint;
+            logger.trace("updated global checkpoint to [{}]", computedGlobalCheckpoint);
+            onGlobalCheckpointUpdated.accept(computedGlobalCheckpoint);
         }
     }
 
@@ -766,8 +796,10 @@ public synchronized void completeRelocationHandoff() {
         assert invariant();
         assert primaryMode;
         assert handoffInProgress;
+        assert relocated == false;
         primaryMode = false;
         handoffInProgress = false;
+        relocated = true;
         // forget all checkpoint information except for global checkpoint of current shard
         checkpoints.entrySet().stream().forEach(e -> {
             final CheckpointState cps = e.getValue();
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index d4a1d0502d0aa..08a0111fb4dc5 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -52,6 +52,7 @@
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -192,7 +193,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
 
     protected volatile ShardRouting shardRouting;
     protected volatile IndexShardState state;
-    protected volatile long primaryTerm;
+    protected volatile long pendingPrimaryTerm; // see JavaDocs for getPendingPrimaryTerm
+    protected volatile long operationPrimaryTerm;
     protected final AtomicReference<Engine> currentEngineReference = new AtomicReference<>();
     final EngineFactory engineFactory;
 
@@ -295,8 +297,9 @@ public IndexShard(
 
         this.checkIndexOnStartup = indexSettings.getValue(IndexSettings.INDEX_CHECK_ON_STARTUP);
         this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, bigArrays);
-        this.replicationTracker = new ReplicationTracker(shardId, shardRouting.allocationId().getId(), indexSettings,
-            SequenceNumbers.UNASSIGNED_SEQ_NO);
+        final String aId = shardRouting.allocationId().getId();
+        this.replicationTracker =
+                new ReplicationTracker(shardId, aId, indexSettings, SequenceNumbers.UNASSIGNED_SEQ_NO, globalCheckpoint -> {});
         // the query cache is a node-level thing, however we want the most popular filters
         // to be computed on a per-shard basis
         if (IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.get(settings)) {
@@ -315,7 +318,8 @@ public boolean shouldCache(Query query) {
         }
         indexShardOperationPermits = new IndexShardOperationPermits(shardId, threadPool);
         searcherWrapper = indexSearcherWrapper;
-        primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id());
+        pendingPrimaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id());
+        operationPrimaryTerm = pendingPrimaryTerm;
         refreshListeners = buildRefreshListeners();
         lastSearcherAccess.set(threadPool.relativeTimeInMillis());
         persistMetadata(path, indexSettings, shardRouting, null, logger);
@@ -365,10 +369,14 @@ public ShardFieldData fieldData() {
     }
 
     /**
-     * Returns the primary term the index shard is on. See {@link org.elasticsearch.cluster.metadata.IndexMetaData#primaryTerm(int)}
+     * USE THIS METHOD WITH CARE!
+     * Returns the primary term the index shard is supposed to be on. In case of primary promotion or when a replica learns about
+     * a new term due to a new primary, the term that's exposed here will not be the term that the shard internally uses to assign
+     * to operations. The shard will auto-correct its internal operation term, but this might take time.
+     * See {@link org.elasticsearch.cluster.metadata.IndexMetaData#primaryTerm(int)}
      */
-    public long getPrimaryTerm() {
-        return this.primaryTerm;
+    public long getPendingPrimaryTerm() {
+        return this.pendingPrimaryTerm;
     }
 
     /**
@@ -418,7 +426,7 @@ public void updateShardState(final ShardRouting newRouting,
                     "a primary relocation is completed by the master, but primary mode is not active " + currentRouting;
 
                 changeState(IndexShardState.STARTED, "global state is [" + newRouting.state() + "]");
-            } else if (currentRouting.primary() && currentRouting.relocating() && replicationTracker.isPrimaryMode() == false &&
+            } else if (currentRouting.primary() && currentRouting.relocating() && replicationTracker.isRelocated() &&
                 (newRouting.relocating() == false || newRouting.equalsIgnoringMetaData(currentRouting) == false)) {
                 // if the shard is not in primary mode anymore (after primary relocation) we have to fail when any changes in shard routing occur (e.g. due to recovery
                 // failure / cancellation). The reason is that at the moment we cannot safely reactivate primary mode without risking two
@@ -431,7 +439,7 @@ public void updateShardState(final ShardRouting newRouting,
             final CountDownLatch shardStateUpdated = new CountDownLatch(1);
 
             if (newRouting.primary()) {
-                if (newPrimaryTerm == primaryTerm) {
+                if (newPrimaryTerm == pendingPrimaryTerm) {
                     if (currentRouting.initializing() && currentRouting.isRelocationTarget() == false && newRouting.active()) {
                         // the master started a recovering primary, activate primary mode.
                         replicationTracker.activatePrimaryMode(getLocalCheckpoint());
@@ -454,10 +462,10 @@ public void updateShardState(final ShardRouting newRouting,
                     assert newRouting.initializing() == false :
                         "a started primary shard should never update its term; "
                             + "shard " + newRouting + ", "
-                            + "current term [" + primaryTerm + "], "
+                            + "current term [" + pendingPrimaryTerm + "], "
                             + "new term [" + newPrimaryTerm + "]";
-                    assert newPrimaryTerm > primaryTerm :
-                        "primary terms can only go up; current term [" + primaryTerm + "], new term [" + newPrimaryTerm + "]";
+                    assert newPrimaryTerm > pendingPrimaryTerm :
+                        "primary terms can only go up; current term [" + pendingPrimaryTerm + "], new term [" + newPrimaryTerm + "]";
                     /*
                      * Before this call returns, we are guaranteed that all future operations are delayed and so this happens before we
                      * increment the primary term. The latch is needed to ensure that we do not unblock operations before the primary term is
@@ -468,12 +476,15 @@ public void updateShardState(final ShardRouting newRouting,
                     if (resyncStarted == false) {
                         throw new IllegalStateException("cannot start resync while it's already in progress");
                     }
-                    indexShardOperationPermits.asyncBlockOperations(
-                        30,
-                        TimeUnit.MINUTES,
+                    bumpPrimaryTerm(newPrimaryTerm,
                         () -> {
                             shardStateUpdated.await();
+                            assert pendingPrimaryTerm == newPrimaryTerm :
+                                "shard term changed on primary. expected [" + newPrimaryTerm + "] but was [" + pendingPrimaryTerm + "]" +
+                                ", current routing: " + currentRouting + ", new routing: " + newRouting;
+                            assert operationPrimaryTerm == newPrimaryTerm;
                             try {
+                                replicationTracker.activatePrimaryMode(getLocalCheckpoint());
                                 /*
                                  * If this shard was serving as a replica shard when another shard was promoted to primary then the state of
                                  * its local checkpoint tracker was reset during the primary term transition. In particular, the local
@@ -517,10 +528,7 @@ public void onFailure(Exception e) {
                             } catch (final AlreadyClosedException e) {
                                 // okay, the index was deleted
                             }
-                        },
-                        e -> failShard("exception during primary term transition", e));
-                    replicationTracker.activatePrimaryMode(getLocalCheckpoint());
-                    primaryTerm = newPrimaryTerm;
+                        });
                 }
             }
             // set this last, once we finished updating all internal state.
@@ -528,8 +536,9 @@ public void onFailure(Exception e) {
 
             assert this.shardRouting.primary() == false ||
                 this.shardRouting.started() == false || // note that we use started and not active to avoid relocating shards
+                this.indexShardOperationPermits.isBlocked() || // if permits are blocked, we are still transitioning
                 this.replicationTracker.isPrimaryMode()
-                : "an started primary must be in primary mode " + this.shardRouting;
+                : "a started primary with non-pending operation term must be in primary mode " + this.shardRouting;
             shardStateUpdated.countDown();
         }
         if (currentRouting != null && currentRouting.active() == false && newRouting.active()) {
@@ -590,7 +599,7 @@ public void relocated(final Consumer<ReplicationTracker.PrimaryContext> consumer
                     consumer.accept(primaryContext);
                     synchronized (mutex) {
                         verifyRelocatingState();
-                        replicationTracker.completeRelocationHandoff(); // make changes to primaryMode flag only under mutex
+                        replicationTracker.completeRelocationHandoff(); // make changes to primaryMode and relocated flag only under mutex
                     }
                 } catch (final Exception e) {
                     try {
@@ -655,21 +664,22 @@ private IndexShardState changeState(IndexShardState newState, String reason) {
     public Engine.IndexResult applyIndexOperationOnPrimary(long version, VersionType versionType, SourceToParse sourceToParse,
                                                            long autoGeneratedTimestamp, boolean isRetry) throws IOException {
         assert versionType.validateVersionForWrites(version);
-        return applyIndexOperation(SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm, version, versionType, autoGeneratedTimestamp,
+        return applyIndexOperation(SequenceNumbers.UNASSIGNED_SEQ_NO, operationPrimaryTerm, version, versionType, autoGeneratedTimestamp,
             isRetry, Engine.Operation.Origin.PRIMARY, sourceToParse);
     }
 
     public Engine.IndexResult applyIndexOperationOnReplica(long seqNo, long version, long autoGeneratedTimeStamp,
                                                            boolean isRetry, SourceToParse sourceToParse)
         throws IOException {
-        return applyIndexOperation(seqNo, primaryTerm, version, null, autoGeneratedTimeStamp, isRetry,
+        return applyIndexOperation(seqNo, operationPrimaryTerm, version, null, autoGeneratedTimeStamp, isRetry,
             Engine.Operation.Origin.REPLICA, sourceToParse);
     }
 
     private Engine.IndexResult applyIndexOperation(long seqNo, long opPrimaryTerm, long version, @Nullable VersionType versionType,
                                                    long autoGeneratedTimeStamp, boolean isRetry, Engine.Operation.Origin origin,
                                                    SourceToParse sourceToParse) throws IOException {
-        assert opPrimaryTerm <= this.primaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.primaryTerm + "]";
+        assert opPrimaryTerm <= this.operationPrimaryTerm: "op term [ " + opPrimaryTerm + " ] > shard term [" + this.operationPrimaryTerm
+            + "]";
         ensureWriteAllowed(origin);
         Engine.Index operation;
         try {
@@ -686,7 +696,7 @@ private Engine.IndexResult applyIndexOperation(long seqNo, long opPrimaryTerm, l
             // can not raise an exception that may block any replication of previous operations to the
             // replicas
             verifyNotClosed(e);
-            return new Engine.IndexResult(e, version, seqNo);
+            return new Engine.IndexResult(e, version, opPrimaryTerm, seqNo);
         }
 
         return index(getEngine(), operation);
@@ -723,12 +733,13 @@ private Engine.IndexResult index(Engine engine, Engine.Index index) throws IOExc
     }
 
     public Engine.NoOpResult markSeqNoAsNoop(long seqNo, String reason) throws IOException {
-        return markSeqNoAsNoop(seqNo, primaryTerm, reason, Engine.Operation.Origin.REPLICA);
+        return markSeqNoAsNoop(seqNo, operationPrimaryTerm, reason, Engine.Operation.Origin.REPLICA);
     }
 
     private Engine.NoOpResult markSeqNoAsNoop(long seqNo, long opPrimaryTerm, String reason,
                                               Engine.Operation.Origin origin) throws IOException {
-        assert opPrimaryTerm <= this.primaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.primaryTerm + "]";
+        assert opPrimaryTerm <= this.operationPrimaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.operationPrimaryTerm
+            + "]";
         long startTime = System.nanoTime();
         ensureWriteAllowed(origin);
         final Engine.NoOp noOp = new Engine.NoOp(seqNo, opPrimaryTerm, origin, startTime, reason);
@@ -743,20 +754,29 @@ private Engine.NoOpResult noOp(Engine engine, Engine.NoOp noOp) {
         return engine.noOp(noOp);
     }
 
+    public Engine.IndexResult getFailedIndexResult(Exception e, long version) {
+        return new Engine.IndexResult(e, version, operationPrimaryTerm);
+    }
+
+    public Engine.DeleteResult getFailedDeleteResult(Exception e, long version) {
+        return new Engine.DeleteResult(e, version, operationPrimaryTerm);
+    }
+
     public Engine.DeleteResult applyDeleteOperationOnPrimary(long version, String type, String id, VersionType versionType)
         throws IOException {
         assert versionType.validateVersionForWrites(version);
-        return applyDeleteOperation(SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm, version, type, id, versionType,
+        return applyDeleteOperation(SequenceNumbers.UNASSIGNED_SEQ_NO, operationPrimaryTerm, version, type, id, versionType,
             Engine.Operation.Origin.PRIMARY);
     }
 
     public Engine.DeleteResult applyDeleteOperationOnReplica(long seqNo, long version, String type, String id) throws IOException {
-        return applyDeleteOperation(seqNo, primaryTerm, version, type, id, null, Engine.Operation.Origin.REPLICA);
+        return applyDeleteOperation(seqNo, operationPrimaryTerm, version, type, id, null, Engine.Operation.Origin.REPLICA);
     }
 
     private Engine.DeleteResult applyDeleteOperation(long seqNo, long opPrimaryTerm, long version, String type, String id,
                                                      @Nullable VersionType versionType, Engine.Operation.Origin origin) throws IOException {
-        assert opPrimaryTerm <= this.primaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.primaryTerm + "]";
+        assert opPrimaryTerm <= this.operationPrimaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.operationPrimaryTerm
+            + "]";
         ensureWriteAllowed(origin);
         // When there is a single type, the unique identifier is only composed of the _id,
         // so there is no way to differenciate foo#1 from bar#1. This is especially an issue
@@ -772,7 +792,7 @@ private Engine.DeleteResult applyDeleteOperation(long seqNo, long opPrimaryTerm,
                 return new Engine.DeleteResult(update);
             }
         } catch (MapperParsingException | IllegalArgumentException | TypeMissingException e) {
-            return new Engine.DeleteResult(e, version, seqNo, false);
+            return new Engine.DeleteResult(e, version, operationPrimaryTerm, seqNo, false);
         }
         final Term uid = extractUidForDelete(type, id);
         final Engine.Delete delete = prepareDelete(type, id, uid, seqNo, opPrimaryTerm, version,
@@ -1209,7 +1229,7 @@ public void prepareForIndexRecovery() {
     }
 
     public void trimOperationOfPreviousPrimaryTerms(long aboveSeqNo) {
-        getEngine().trimOperationsFromTranslog(primaryTerm, aboveSeqNo);
+        getEngine().trimOperationsFromTranslog(operationPrimaryTerm, aboveSeqNo);
     }
 
     public Engine.Result applyTranslogOperation(Translog.Operation operation, Engine.Operation.Origin origin) throws IOException {
@@ -1427,10 +1447,10 @@ private void ensureWriteAllowed(Engine.Operation.Origin origin) throws IllegalIn
             }
         } else {
             if (origin == Engine.Operation.Origin.PRIMARY) {
-                verifyPrimary();
+                assert assertPrimaryMode();
             } else {
                 assert origin == Engine.Operation.Origin.REPLICA;
-                verifyReplicationTarget();
+                assert assertReplicationTarget();
             }
             if (writeAllowedStates.contains(state) == false) {
                 throw new IllegalIndexShardStateException(shardId, state, "operation only allowed when shard state is one of " + writeAllowedStates + ", origin [" + origin + "]");
@@ -1438,19 +1458,14 @@ private void ensureWriteAllowed(Engine.Operation.Origin origin) throws IllegalIn
         }
     }
 
-    private void verifyPrimary() {
-        if (shardRouting.primary() == false) {
-            throw new IllegalStateException("shard " + shardRouting + " is not a primary");
-        }
+    private boolean assertPrimaryMode() {
+        assert shardRouting.primary() && replicationTracker.isPrimaryMode() : "shard " + shardRouting + " is not a primary shard in primary mode";
+        return true;
     }
 
-    private void verifyReplicationTarget() {
-        final IndexShardState state = state();
-        if (shardRouting.primary() && shardRouting.active() && replicationTracker.isPrimaryMode()) {
-            // must use exception that is not ignored by replication logic. See TransportActions.isShardNotAvailableException
-            throw new IllegalStateException("active primary shard " + shardRouting + " cannot be a replication target before " +
-                "relocation hand off, state is [" + state + "]");
-        }
+    private boolean assertReplicationTarget() {
+        assert replicationTracker.isPrimaryMode() == false : "shard " + shardRouting + " in primary mode cannot be a replication target";
+        return true;
     }
 
     private void verifyNotClosed() throws IllegalIndexShardStateException {
@@ -1697,7 +1712,7 @@ public void writeIndexingBuffer() {
      * @param checkpoint   the local checkpoint for the shard
      */
     public void updateLocalCheckpointForShard(final String allocationId, final long checkpoint) {
-        verifyPrimary();
+        assert assertPrimaryMode();
         verifyNotClosed();
         replicationTracker.updateLocalCheckpoint(allocationId, checkpoint);
     }
@@ -1709,7 +1724,7 @@ public void updateLocalCheckpointForShard(final String allocationId, final long
      * @param globalCheckpoint the global checkpoint
      */
     public void updateGlobalCheckpointForShard(final String allocationId, final long globalCheckpoint) {
-        verifyPrimary();
+        assert assertPrimaryMode();
         verifyNotClosed();
         replicationTracker.updateGlobalCheckpointForShard(allocationId, globalCheckpoint);
     }
@@ -1731,7 +1746,7 @@ public void waitForOpsToComplete(final long seqNo) throws InterruptedException {
      * @param allocationId  the allocation ID of the shard for which recovery was initiated
      */
     public void initiateTracking(final String allocationId) {
-        verifyPrimary();
+        assert assertPrimaryMode();
         replicationTracker.initiateTracking(allocationId);
     }
 
@@ -1744,7 +1759,7 @@ public void initiateTracking(final String allocationId) {
      * @param localCheckpoint the current local checkpoint on the shard
      */
     public void markAllocationIdAsInSync(final String allocationId, final long localCheckpoint) throws InterruptedException {
-        verifyPrimary();
+        assert assertPrimaryMode();
         replicationTracker.markAllocationIdAsInSync(allocationId, localCheckpoint);
     }
 
@@ -1779,7 +1794,7 @@ public long getLastSyncedGlobalCheckpoint() {
      * @return a map from allocation ID to the local knowledge of the global checkpoint for that allocation ID
      */
     public ObjectLongMap<String> getInSyncGlobalCheckpoints() {
-        verifyPrimary();
+        assert assertPrimaryMode();
         verifyNotClosed();
         return replicationTracker.getInSyncGlobalCheckpoints();
     }
@@ -1789,11 +1804,12 @@ public ObjectLongMap<String> getInSyncGlobalCheckpoints() {
      * primary.
      */
     public void maybeSyncGlobalCheckpoint(final String reason) {
-        verifyPrimary();
         verifyNotClosed();
+        assert shardRouting.primary() : "only call maybeSyncGlobalCheckpoint on primary shard";
         if (replicationTracker.isPrimaryMode() == false) {
             return;
         }
+        assert assertPrimaryMode();
         // only sync if there are not operations in flight
         final SeqNoStats stats = getEngine().getSeqNoStats(replicationTracker.getGlobalCheckpoint());
         if (stats.getMaxSeqNo() == stats.getGlobalCheckpoint()) {
@@ -1819,7 +1835,7 @@ public void maybeSyncGlobalCheckpoint(final String reason) {
      * @return the replication group
      */
     public ReplicationGroup getReplicationGroup() {
-        verifyPrimary();
+        assert assertPrimaryMode();
         verifyNotClosed();
         return replicationTracker.getReplicationGroup();
     }
@@ -1831,7 +1847,7 @@ public ReplicationGroup getReplicationGroup() {
      * @param reason           the reason the global checkpoint was updated
      */
     public void updateGlobalCheckpointOnReplica(final long globalCheckpoint, final String reason) {
-        verifyReplicationTarget();
+        assert assertReplicationTarget();
         final long localCheckpoint = getLocalCheckpoint();
         if (globalCheckpoint > localCheckpoint) {
             /*
@@ -1858,8 +1874,7 @@ assert state() != IndexShardState.POST_RECOVERY && state() != IndexShardState.ST
      * @param primaryContext the sequence number context
      */
     public void activateWithPrimaryContext(final ReplicationTracker.PrimaryContext primaryContext) {
-        verifyPrimary();
-        assert shardRouting.isRelocationTarget() : "only relocation target can update allocation IDs from primary context: " + shardRouting;
+        assert shardRouting.primary() && shardRouting.isRelocationTarget() : "only primary relocation target can update allocation IDs from primary context: " + shardRouting;
         assert primaryContext.getCheckpointStates().containsKey(routingEntry().allocationId().getId()) &&
             getLocalCheckpoint() == primaryContext.getCheckpointStates().get(routingEntry().allocationId().getId()).getLocalCheckpoint();
         synchronized (mutex) {
@@ -1873,7 +1888,7 @@ public void activateWithPrimaryContext(final ReplicationTracker.PrimaryContext p
      * @return {@code true} if there is at least one shard pending in-sync, otherwise false
      */
     public boolean pendingInSync() {
-        verifyPrimary();
+        assert assertPrimaryMode();
         return replicationTracker.pendingInSync();
     }
 
@@ -2082,10 +2097,11 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService
     }
 
     /**
-     * Returns whether the shard is in primary mode, i.e., in charge of replicating changes (see {@link ReplicationTracker}).
+     * Returns whether the shard is a relocated primary, i.e. not in charge anymore of replicating changes (see {@link ReplicationTracker}).
      */
-    public boolean isPrimaryMode() {
-        return replicationTracker.isPrimaryMode();
+    public boolean isRelocatedPrimary() {
+        assert shardRouting.primary() : "only call isRelocatedPrimary on primary shard";
+        return replicationTracker.isRelocated();
     }
 
     class ShardEventListener implements Engine.EventListener {
@@ -2175,7 +2191,7 @@ private EngineConfig newEngineConfig() {
             IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()),
             Collections.singletonList(refreshListeners),
             Collections.singletonList(new RefreshMetricUpdater(refreshMetric)),
-            indexSort, this::runTranslogRecovery, circuitBreakerService, replicationTracker, this::getPrimaryTerm);
+            indexSort, this::runTranslogRecovery, circuitBreakerService, replicationTracker, () -> operationPrimaryTerm);
     }
 
     /**
@@ -2189,12 +2205,36 @@ private EngineConfig newEngineConfig() {
      */
     public void acquirePrimaryOperationPermit(ActionListener<Releasable> onPermitAcquired, String executorOnDelay, Object debugInfo) {
         verifyNotClosed();
-        verifyPrimary();
+        assert shardRouting.primary() : "acquirePrimaryOperationPermit should only be called on primary shard: " + shardRouting;
 
         indexShardOperationPermits.acquire(onPermitAcquired, executorOnDelay, false, debugInfo);
     }
 
-    private final Object primaryTermMutex = new Object();
+    private <E extends Exception> void bumpPrimaryTerm(long newPrimaryTerm, final CheckedRunnable<E> onBlocked) {
+        assert Thread.holdsLock(mutex);
+        assert newPrimaryTerm > pendingPrimaryTerm;
+        assert operationPrimaryTerm <= pendingPrimaryTerm;
+        final CountDownLatch termUpdated = new CountDownLatch(1);
+        indexShardOperationPermits.asyncBlockOperations(30, TimeUnit.MINUTES, () -> {
+                assert operationPrimaryTerm <= pendingPrimaryTerm;
+                termUpdated.await();
+                // indexShardOperationPermits doesn't guarantee that async submissions are executed
+                // in the order submitted. We need to guard against another term bump
+                if (operationPrimaryTerm < newPrimaryTerm) {
+                    operationPrimaryTerm = newPrimaryTerm;
+                    onBlocked.run();
+                }
+            },
+            e -> {
+                try {
+                    failShard("exception during primary term transition", e);
+                } catch (AlreadyClosedException ace) {
+                    // ignore, shard is already closed
+                }
+            });
+        pendingPrimaryTerm = newPrimaryTerm;
+        termUpdated.countDown();
+    }
 
     /**
      * Acquire a replica operation permit whenever the shard is ready for indexing (see
@@ -2203,7 +2243,7 @@ public void acquirePrimaryOperationPermit(ActionListener<Releasable> onPermitAcq
      * {@link IllegalStateException}. If permit acquisition is delayed, the listener will be invoked on the executor with the specified
      * name.
      *
-     * @param operationPrimaryTerm the operation primary term
+     * @param opPrimaryTerm        the operation primary term
      * @param globalCheckpoint     the global checkpoint associated with the request
      * @param onPermitAcquired     the listener for permit acquisition
      * @param executorOnDelay      the name of the executor to invoke the listener on if permit acquisition is delayed
@@ -2211,15 +2251,13 @@ public void acquirePrimaryOperationPermit(ActionListener<Releasable> onPermitAcq
      *                             the tracing will capture the supplied object's {@link Object#toString()} value. Otherwise the object
      *                             isn't used
      */
-    public void acquireReplicaOperationPermit(final long operationPrimaryTerm, final long globalCheckpoint,
+    public void acquireReplicaOperationPermit(final long opPrimaryTerm, final long globalCheckpoint,
                                               final ActionListener<Releasable> onPermitAcquired, final String executorOnDelay,
                                               final Object debugInfo) {
         verifyNotClosed();
-        verifyReplicationTarget();
-        final boolean globalCheckpointUpdated;
-        if (operationPrimaryTerm > primaryTerm) {
-            synchronized (primaryTermMutex) {
-                if (operationPrimaryTerm > primaryTerm) {
+        if (opPrimaryTerm > pendingPrimaryTerm) {
+            synchronized (mutex) {
+                if (opPrimaryTerm > pendingPrimaryTerm) {
                     IndexShardState shardState = state();
                     // only roll translog and update primary term if shard has made it past recovery
                     // Having a new primary term here means that the old primary failed and that there is a new primary, which again
@@ -2229,64 +2267,53 @@ public void acquireReplicaOperationPermit(final long operationPrimaryTerm, final
                         shardState != IndexShardState.STARTED) {
                         throw new IndexShardNotStartedException(shardId, shardState);
                     }
-                    try {
-                        indexShardOperationPermits.blockOperations(30, TimeUnit.MINUTES, () -> {
-                            assert operationPrimaryTerm > primaryTerm :
-                                "shard term already update.  op term [" + operationPrimaryTerm + "], shardTerm [" + primaryTerm + "]";
-                            primaryTerm = operationPrimaryTerm;
-                            updateGlobalCheckpointOnReplica(globalCheckpoint, "primary term transition");
-                            final long currentGlobalCheckpoint = getGlobalCheckpoint();
-                            final long localCheckpoint;
-                            if (currentGlobalCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO) {
-                                localCheckpoint = SequenceNumbers.NO_OPS_PERFORMED;
-                            } else {
-                                localCheckpoint = currentGlobalCheckpoint;
-                            }
-                            logger.trace(
+
+                    if (opPrimaryTerm > pendingPrimaryTerm) {
+                        bumpPrimaryTerm(opPrimaryTerm, () -> {
+                                updateGlobalCheckpointOnReplica(globalCheckpoint, "primary term transition");
+                                final long currentGlobalCheckpoint = getGlobalCheckpoint();
+                                final long localCheckpoint;
+                                if (currentGlobalCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO) {
+                                    localCheckpoint = SequenceNumbers.NO_OPS_PERFORMED;
+                                } else {
+                                    localCheckpoint = currentGlobalCheckpoint;
+                                }
+                                logger.trace(
                                     "detected new primary with primary term [{}], resetting local checkpoint from [{}] to [{}]",
-                                    operationPrimaryTerm,
+                                    opPrimaryTerm,
                                     getLocalCheckpoint(),
                                     localCheckpoint);
-                            getEngine().resetLocalCheckpoint(localCheckpoint);
-                            getEngine().rollTranslogGeneration();
+                                getEngine().resetLocalCheckpoint(localCheckpoint);
+                                getEngine().rollTranslogGeneration();
                         });
-                        globalCheckpointUpdated = true;
-                    } catch (final Exception e) {
-                        onPermitAcquired.onFailure(e);
-                        return;
                     }
-                } else {
-                    globalCheckpointUpdated = false;
                 }
             }
-        } else {
-            globalCheckpointUpdated = false;
         }
 
-        assert operationPrimaryTerm <= primaryTerm
-                : "operation primary term [" + operationPrimaryTerm + "] should be at most [" + primaryTerm + "]";
+        assert opPrimaryTerm <= pendingPrimaryTerm
+                : "operation primary term [" + opPrimaryTerm + "] should be at most [" + pendingPrimaryTerm + "]";
         indexShardOperationPermits.acquire(
                 new ActionListener<Releasable>() {
                     @Override
                     public void onResponse(final Releasable releasable) {
-                        if (operationPrimaryTerm < primaryTerm) {
+                        if (opPrimaryTerm < operationPrimaryTerm) {
                             releasable.close();
                             final String message = String.format(
                                     Locale.ROOT,
                                     "%s operation primary term [%d] is too old (current [%d])",
                                     shardId,
-                                    operationPrimaryTerm,
-                                    primaryTerm);
+                                    opPrimaryTerm,
+                                    operationPrimaryTerm);
                             onPermitAcquired.onFailure(new IllegalStateException(message));
                         } else {
-                            if (globalCheckpointUpdated == false) {
-                                try {
-                                    updateGlobalCheckpointOnReplica(globalCheckpoint, "operation");
-                                } catch (Exception e) {
-                                    releasable.close();
-                                    onPermitAcquired.onFailure(e);
-                                    return;
-                                }
+                            assert assertReplicationTarget();
+                            try {
+                                updateGlobalCheckpointOnReplica(globalCheckpoint, "operation");
+                            } catch (Exception e) {
+                                releasable.close();
+                                onPermitAcquired.onFailure(e);
+                                return;
                             }
                             onPermitAcquired.onResponse(releasable);
                         }
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java
index a0d46c8eb2330..d4c3833b13a58 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.shard;
 
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.Assertions;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
@@ -29,10 +28,12 @@
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -59,7 +60,7 @@ final class IndexShardOperationPermits implements Closeable {
     final Semaphore semaphore = new Semaphore(TOTAL_PERMITS, true); // fair to ensure a blocking thread is not starved
     private final List<DelayedOperation> delayedOperations = new ArrayList<>(); // operations that are delayed
     private volatile boolean closed;
-    private boolean delayed; // does not need to be volatile as all accesses are done under a lock on this
+    private int queuedBlockOperations; // does not need to be volatile as all accesses are done under a lock on this
 
     // only valid when assertions are enabled. Key is AtomicBoolean associated with each permit to ensure close once semantics.
     // Value is a tuple, with a some debug information supplied by the caller and a stack trace of the acquiring thread
@@ -102,9 +103,6 @@ <E extends Exception> void blockOperations(
             final long timeout,
             final TimeUnit timeUnit,
             final CheckedRunnable<E> onBlocked) throws InterruptedException, TimeoutException, E {
-        if (closed) {
-            throw new IndexShardClosedException(shardId);
-        }
         delayOperations();
         try {
             doBlockOperations(timeout, timeUnit, onBlocked);
@@ -147,13 +145,12 @@ public void onAfter() {
     }
 
     private void delayOperations() {
+        if (closed) {
+            throw new IndexShardClosedException(shardId);
+        }
         synchronized (this) {
-            if (delayed) {
-                throw new IllegalStateException("operations are already delayed");
-            } else {
-                assert delayedOperations.isEmpty();
-                delayed = true;
-            }
+            assert queuedBlockOperations > 0 || delayedOperations.isEmpty();
+            queuedBlockOperations++;
         }
     }
 
@@ -164,7 +161,7 @@ private <E extends Exception> void doBlockOperations(
         if (Assertions.ENABLED) {
             // since delayed is not volatile, we have to synchronize even here for visibility
             synchronized (this) {
-                assert delayed;
+                assert queuedBlockOperations > 0;
             }
         }
         if (semaphore.tryAcquire(TOTAL_PERMITS, timeout, timeUnit)) {
@@ -182,10 +179,14 @@ private <E extends Exception> void doBlockOperations(
     private void releaseDelayedOperations() {
         final List<DelayedOperation> queuedActions;
         synchronized (this) {
-            assert delayed;
-            queuedActions = new ArrayList<>(delayedOperations);
-            delayedOperations.clear();
-            delayed = false;
+            assert queuedBlockOperations > 0;
+            queuedBlockOperations--;
+            if (queuedBlockOperations == 0) {
+                queuedActions = new ArrayList<>(delayedOperations);
+                delayedOperations.clear();
+            } else {
+                queuedActions = Collections.emptyList();
+            }
         }
         if (!queuedActions.isEmpty()) {
             /*
@@ -242,7 +243,7 @@ private void acquire(final ActionListener<Releasable> onAcquired, final String e
         final Releasable releasable;
         try {
             synchronized (this) {
-                if (delayed) {
+                if (queuedBlockOperations > 0) {
                     final Supplier<StoredContext> contextSupplier = threadPool.getThreadContext().newRestorableContext(false);
                     final ActionListener<Releasable> wrappedListener;
                     if (executorOnDelay != null) {
@@ -308,6 +309,11 @@ int getActiveOperationsCount() {
         }
     }
 
+
+    synchronized boolean isBlocked() {
+        return queuedBlockOperations > 0;
+    }
+
     /**
      * @return a list of describing each permit that wasn't released yet. The description consist of the debugInfo supplied
      *         when the permit was acquired plus a stack traces that was captured when the permit was request.
diff --git a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
index e66d78f2e1a05..1edc0eb5dcafe 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
@@ -136,7 +136,7 @@ public void onFailure(final Exception e) {
                 }
             };
 
-            resync(shardId, indexShard.routingEntry().allocationId().getId(), indexShard.getPrimaryTerm(), wrappedSnapshot,
+            resync(shardId, indexShard.routingEntry().allocationId().getId(), indexShard.getPendingPrimaryTerm(), wrappedSnapshot,
                 startingSeqNo, maxSeqNo, resyncListener);
         } catch (Exception e) {
             try {
diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index 54718c545a44e..e9acfe3d8b06f 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -394,7 +394,7 @@ private void internalRecoverFromStore(IndexShard indexShard) throws IndexShardRe
                 final SegmentInfos segmentInfos = store.readLastCommittedSegmentsInfo();
                 final long maxSeqNo = Long.parseLong(segmentInfos.userData.get(SequenceNumbers.MAX_SEQ_NO));
                 final String translogUUID = Translog.createEmptyTranslog(
-                    indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPrimaryTerm());
+                    indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPendingPrimaryTerm());
                 store.associateIndexWithNewTranslog(translogUUID);
             } else if (indexShouldExists) {
                 // since we recover from local, just fill the files and size
@@ -409,11 +409,12 @@ private void internalRecoverFromStore(IndexShard indexShard) throws IndexShardRe
             } else {
                 store.createEmpty();
                 final String translogUUID = Translog.createEmptyTranslog(
-                    indexShard.shardPath().resolveTranslog(), SequenceNumbers.NO_OPS_PERFORMED, shardId, indexShard.getPrimaryTerm());
+                    indexShard.shardPath().resolveTranslog(), SequenceNumbers.NO_OPS_PERFORMED, shardId,
+                    indexShard.getPendingPrimaryTerm());
                 store.associateIndexWithNewTranslog(translogUUID);
             }
             indexShard.openEngineAndRecoverFromTranslog();
-            indexShard.getEngine().fillSeqNoGaps(indexShard.getPrimaryTerm());
+            indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
             indexShard.finalizeRecovery();
             indexShard.postRecovery("post recovery from shard_store");
         } catch (EngineException | IOException e) {
@@ -458,11 +459,11 @@ private void restore(final IndexShard indexShard, final Repository repository, f
             final SegmentInfos segmentInfos = store.readLastCommittedSegmentsInfo();
             final long maxSeqNo = Long.parseLong(segmentInfos.userData.get(SequenceNumbers.MAX_SEQ_NO));
             final String translogUUID = Translog.createEmptyTranslog(
-                indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPrimaryTerm());
+                indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPendingPrimaryTerm());
             store.associateIndexWithNewTranslog(translogUUID);
             assert indexShard.shardRouting.primary() : "only primary shards can recover from store";
             indexShard.openEngineAndRecoverFromTranslog();
-            indexShard.getEngine().fillSeqNoGaps(indexShard.getPrimaryTerm());
+            indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
             indexShard.finalizeRecovery();
             indexShard.postRecovery("restore done");
         } catch (Exception e) {
diff --git a/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java b/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
index ff226ae00bef5..41c3252eab07a 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
@@ -79,7 +79,9 @@ protected final int readSize(ByteBuffer reusableBuffer, long position) throws IO
         final int size = reusableBuffer.getInt() + 4;
         final long maxSize = sizeInBytes() - position;
         if (size < 0 || size > maxSize) {
-            throw new TranslogCorruptedException("operation size is corrupted must be [0.." + maxSize + "] but was: " + size);
+            throw new TranslogCorruptedException(
+                    path.toString(),
+                    "operation size is corrupted must be [0.." + maxSize + "] but was: " + size);
         }
         return size;
     }
@@ -103,14 +105,16 @@ protected final BufferedChecksumStreamInput checksummedStream(ByteBuffer reusabl
         buffer.limit(opSize);
         readBytes(buffer, position);
         buffer.flip();
-        return new BufferedChecksumStreamInput(new ByteBufferStreamInput(buffer), reuse);
+        return new BufferedChecksumStreamInput(new ByteBufferStreamInput(buffer), path.toString(), reuse);
     }
 
     protected Translog.Operation read(BufferedChecksumStreamInput inStream) throws IOException {
         final Translog.Operation op = Translog.readOperation(inStream);
         if (op.primaryTerm() > getPrimaryTerm() && getPrimaryTerm() != TranslogHeader.UNKNOWN_PRIMARY_TERM) {
-            throw new TranslogCorruptedException("Operation's term is newer than translog header term; " +
-                "operation term[" + op.primaryTerm() + "], translog header term [" + getPrimaryTerm() + "]");
+            throw new TranslogCorruptedException(
+                    path.toString(),
+                    "operation's term is newer than translog header term; " +
+                    "operation term[" + op.primaryTerm() + "], translog header term [" + getPrimaryTerm() + "]");
         }
         return op;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java b/server/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java
index 37740b460b766..8e815d3599a6b 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java
@@ -35,14 +35,11 @@ public final class BufferedChecksumStreamInput extends FilterStreamInput {
     private static final int SKIP_BUFFER_SIZE = 1024;
     private byte[] skipBuffer;
     private final Checksum digest;
+    private final String source;
 
-    public BufferedChecksumStreamInput(StreamInput in) {
-        super(in);
-        this.digest = new BufferedChecksum(new CRC32());
-    }
-
-    public BufferedChecksumStreamInput(StreamInput in, BufferedChecksumStreamInput reuse) {
+    public BufferedChecksumStreamInput(StreamInput in, String source, BufferedChecksumStreamInput reuse) {
         super(in);
+        this.source = source;
         if (reuse == null ) {
             this.digest = new BufferedChecksum(new CRC32());
         } else {
@@ -52,6 +49,10 @@ public BufferedChecksumStreamInput(StreamInput in, BufferedChecksumStreamInput r
         }
     }
 
+    public BufferedChecksumStreamInput(StreamInput in, String source) {
+        this(in, source, null);
+    }
+
     public long getChecksum() {
         return this.digest.getValue();
     }
@@ -85,7 +86,6 @@ public boolean markSupported() {
         return delegate.markSupported();
     }
 
-
     @Override
     public long skip(long numBytes) throws IOException {
         if (numBytes < 0) {
@@ -104,7 +104,6 @@ public long skip(long numBytes) throws IOException {
         return skipped;
     }
 
-
     @Override
     public synchronized void mark(int readlimit) {
         delegate.mark(readlimit);
@@ -114,4 +113,7 @@ public void resetDigest() {
         digest.reset();
     }
 
+    public String getSource(){
+        return source;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java
index 31404b7874a92..e426b3a7253eb 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -491,7 +491,10 @@ public Location add(final Operation operation) throws IOException {
             try (ReleasableLock ignored = readLock.acquire()) {
                 ensureOpen();
                 if (operation.primaryTerm() > current.getPrimaryTerm()) {
-                    throw new IllegalArgumentException("Operation term is newer than the current term;"
+                    assert false :
+                        "Operation term is newer than the current term; "
+                            + "current term[" + current.getPrimaryTerm() + "], operation term[" + operation + "]";
+                    throw new IllegalArgumentException("Operation term is newer than the current term; "
                         + "current term[" + current.getPrimaryTerm() + "], operation term[" + operation + "]");
                 }
                 return current.add(bytes, operation.seqNo());
@@ -1424,7 +1427,7 @@ static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException {
         long expectedChecksum = in.getChecksum();
         long readChecksum = Integer.toUnsignedLong(in.readInt());
         if (readChecksum != expectedChecksum) {
-            throw new TranslogCorruptedException("translog stream is corrupted, expected: 0x" +
+            throw new TranslogCorruptedException(in.getSource(), "checksum verification failed - expected: 0x" +
                 Long.toHexString(expectedChecksum) + ", got: 0x" + Long.toHexString(readChecksum));
         }
     }
@@ -1432,10 +1435,10 @@ static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException {
     /**
      * Reads a list of operations written with {@link #writeOperations(StreamOutput, List)}
      */
-    public static List<Operation> readOperations(StreamInput input) throws IOException {
+    public static List<Operation> readOperations(StreamInput input, String source) throws IOException {
         ArrayList<Operation> operations = new ArrayList<>();
         int numOps = input.readInt();
-        final BufferedChecksumStreamInput checksumStreamInput = new BufferedChecksumStreamInput(input);
+        final BufferedChecksumStreamInput checksumStreamInput = new BufferedChecksumStreamInput(input, source);
         for (int i = 0; i < numOps; i++) {
             operations.add(readOperation(checksumStreamInput));
         }
@@ -1447,7 +1450,7 @@ static Translog.Operation readOperation(BufferedChecksumStreamInput in) throws I
         try {
             final int opSize = in.readInt();
             if (opSize < 4) { // 4byte for the checksum
-                throw new TranslogCorruptedException("operation size must be at least 4 but was: " + opSize);
+                throw new TranslogCorruptedException(in.getSource(), "operation size must be at least 4 but was: " + opSize);
             }
             in.resetDigest(); // size is not part of the checksum!
             if (in.markSupported()) { // if we can we validate the checksum first
@@ -1462,17 +1465,15 @@ static Translog.Operation readOperation(BufferedChecksumStreamInput in) throws I
             }
             operation = Translog.Operation.readOperation(in);
             verifyChecksum(in);
-        } catch (TranslogCorruptedException e) {
-            throw e;
         } catch (EOFException e) {
-            throw new TruncatedTranslogException("reached premature end of file, translog is truncated", e);
+            throw new TruncatedTranslogException(in.getSource(), "reached premature end of file, translog is truncated", e);
         }
         return operation;
     }
 
     /**
      * Writes all operations in the given iterable to the given output stream including the size of the array
-     * use {@link #readOperations(StreamInput)} to read it back.
+     * use {@link #readOperations(StreamInput, String)} to read it back.
      */
     public static void writeOperations(StreamOutput outStream, List<Operation> toWrite) throws IOException {
         final ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(BigArrays.NON_RECYCLING_INSTANCE);
@@ -1713,7 +1714,7 @@ private static Checkpoint readCheckpoint(Path location, String expectedTranslogU
         } catch (TranslogCorruptedException ex) {
             throw ex; // just bubble up.
         } catch (Exception ex) {
-            throw new TranslogCorruptedException("Translog at [" + location + "] is corrupted", ex);
+            throw new TranslogCorruptedException(location.toString(), ex);
         }
         return checkpoint;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogCorruptedException.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogCorruptedException.java
index 07700b3037cbb..ab1a48b216706 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TranslogCorruptedException.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogCorruptedException.java
@@ -25,15 +25,27 @@
 import java.io.IOException;
 
 public class TranslogCorruptedException extends ElasticsearchException {
-    public TranslogCorruptedException(String msg) {
-        super(msg);
+    public TranslogCorruptedException(String source, String details) {
+        super(corruptedMessage(source, details));
     }
 
-    public TranslogCorruptedException(String msg, Throwable cause) {
-        super(msg, cause);
+    public TranslogCorruptedException(String source, Throwable cause) {
+        this(source, null, cause);
     }
 
-    public TranslogCorruptedException(StreamInput in) throws IOException{
+    public TranslogCorruptedException(String source, String details, Throwable cause) {
+        super(corruptedMessage(source, details), cause);
+    }
+
+    private static String corruptedMessage(String source, String details) {
+        String msg = "translog from source [" + source + "] is corrupted";
+        if (details != null) {
+            msg += ", " + details;
+        }
+        return msg;
+    }
+
+    public TranslogCorruptedException(StreamInput in) throws IOException {
         super(in);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java
index 0fde24d8bb4d5..20aadf21bcb48 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java
@@ -110,13 +110,15 @@ private static int headerSizeInBytes(int version, int uuidLength) {
     static TranslogHeader read(final String translogUUID, final Path path, final FileChannel channel) throws IOException {
         // This input is intentionally not closed because closing it will close the FileChannel.
         final BufferedChecksumStreamInput in =
-            new BufferedChecksumStreamInput(new InputStreamStreamInput(java.nio.channels.Channels.newInputStream(channel), channel.size()));
+            new BufferedChecksumStreamInput(
+                    new InputStreamStreamInput(java.nio.channels.Channels.newInputStream(channel), channel.size()),
+                    path.toString());
         final int version;
         try {
             version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_CHECKSUMS, VERSION_PRIMARY_TERM);
         } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
             tryReportOldVersionError(path, channel);
-            throw new TranslogCorruptedException("Translog header corrupted. path:" + path, e);
+            throw new TranslogCorruptedException(path.toString(), "translog header corrupted", e);
         }
         if (version == VERSION_CHECKSUMS) {
             throw new IllegalStateException("pre-2.0 translog found [" + path + "]");
@@ -124,15 +126,19 @@ static TranslogHeader read(final String translogUUID, final Path path, final Fil
         // Read the translogUUID
         final int uuidLen = in.readInt();
         if (uuidLen > channel.size()) {
-            throw new TranslogCorruptedException("uuid length can't be larger than the translog");
+            throw new TranslogCorruptedException(
+                    path.toString(),
+                    "UUID length can't be larger than the translog");
         }
         final BytesRef uuid = new BytesRef(uuidLen);
         uuid.length = uuidLen;
         in.read(uuid.bytes, uuid.offset, uuid.length);
         final BytesRef expectedUUID = new BytesRef(translogUUID);
         if (uuid.bytesEquals(expectedUUID) == false) {
-            throw new TranslogCorruptedException("expected shard UUID " + expectedUUID + " but got: " + uuid +
-                " this translog file belongs to a different translog. path:" + path);
+            throw new TranslogCorruptedException(
+                    path.toString(),
+                    "expected shard UUID " + expectedUUID + " but got: " + uuid +
+                            " this translog file belongs to a different translog");
         }
         // Read the primary term
         final long primaryTerm;
@@ -164,7 +170,9 @@ private static void tryReportOldVersionError(final Path path, final FileChannel
         // 0x00 => version 0 of the translog
         final byte b1 = Channels.readFromFileChannel(channel, 0, 1)[0];
         if (b1 == 0x3f) { // LUCENE_CODEC_HEADER_BYTE
-            throw new TranslogCorruptedException("translog looks like version 1 or later, but has corrupted header. path:" + path);
+            throw new TranslogCorruptedException(
+                    path.toString(),
+                    "translog looks like version 1 or later, but has corrupted header" );
         } else if (b1 == 0x00) { // UNVERSIONED_TRANSLOG_HEADER_BYTE
             throw new IllegalStateException("pre-1.4 translog found [" + path + "]");
         }
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index c135facc67f5f..b779644cd5c58 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -200,8 +200,10 @@ private synchronized boolean assertNoSeqNumberConflict(long seqNo, BytesReferenc
         } else if (seenSequenceNumbers.containsKey(seqNo)) {
             final Tuple<BytesReference, Exception> previous = seenSequenceNumbers.get(seqNo);
             if (previous.v1().equals(data) == false) {
-                Translog.Operation newOp = Translog.readOperation(new BufferedChecksumStreamInput(data.streamInput()));
-                Translog.Operation prvOp = Translog.readOperation(new BufferedChecksumStreamInput(previous.v1().streamInput()));
+                Translog.Operation newOp = Translog.readOperation(
+                        new BufferedChecksumStreamInput(data.streamInput(), "assertion"));
+                Translog.Operation prvOp = Translog.readOperation(
+                        new BufferedChecksumStreamInput(previous.v1().streamInput(), "assertion"));
                 if (newOp.equals(prvOp) == false) {
                     throw new AssertionError(
                         "seqNo [" + seqNo + "] was processed twice in generation [" + generation + "], with different data. " +
@@ -220,7 +222,8 @@ synchronized boolean assertNoSeqAbove(long belowTerm, long aboveSeqNo) {
             .forEach(e -> {
                 final Translog.Operation op;
                 try {
-                    op = Translog.readOperation(new BufferedChecksumStreamInput(e.getValue().v1().streamInput()));
+                    op = Translog.readOperation(
+                            new BufferedChecksumStreamInput(e.getValue().v1().streamInput(), "assertion"));
                 } catch (IOException ex) {
                     throw new RuntimeException(ex);
                 }
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TruncatedTranslogException.java b/server/src/main/java/org/elasticsearch/index/translog/TruncatedTranslogException.java
index e04eb58068d2f..5e0be02b7fcfc 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TruncatedTranslogException.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TruncatedTranslogException.java
@@ -25,11 +25,12 @@
 
 public class TruncatedTranslogException extends TranslogCorruptedException {
 
-    public TruncatedTranslogException(String msg, Throwable cause) {
-        super(msg, cause);
-    }
-
     public TruncatedTranslogException(StreamInput in) throws IOException {
         super(in);
     }
+
+    public TruncatedTranslogException(String source, String details, Throwable cause) {
+        super(source, details, cause);
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 472cb04936d64..e6a86d47f55c0 100644
--- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -420,6 +420,12 @@ private void removeShards(final ClusterState state) {
                     // state may result in a new shard being initialized while having the same allocation id as the currently started shard.
                     logger.debug("{} removing shard (not active, current {}, new {})", shardId, currentRoutingEntry, newShardRouting);
                     indexService.removeShard(shardId.id(), "removing shard (stale copy)");
+                } else if (newShardRouting.primary() && currentRoutingEntry.primary() == false && newShardRouting.initializing()) {
+                    assert currentRoutingEntry.initializing() : currentRoutingEntry; // see above if clause
+                    // this can happen when cluster state batching batches activation of the shard, closing an index, reopening it
+                    // and assigning an initializing primary to this node
+                    logger.debug("{} removing shard (not active, current {}, new {})", shardId, currentRoutingEntry, newShardRouting);
+                    indexService.removeShard(shardId.id(), "removing shard (stale copy)");
                 }
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index 45500349865f7..352f07d57649d 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -250,7 +250,7 @@ public void onFailure(Exception e) {
             try (Releasable ignored = FutureUtils.get(permit)) {
                 // check that the IndexShard still has the primary authority. This needs to be checked under operation permit to prevent
                 // races, as IndexShard will switch its authority only when it holds all operation permits, see IndexShard.relocated()
-                if (primary.isPrimaryMode() == false) {
+                if (primary.isRelocatedPrimary()) {
                     throw new IndexShardRelocatedException(primary.shardId());
                 }
                 runnable.run();
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
index 57deb4666daca..1a772f0c3f882 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
@@ -443,7 +443,8 @@ public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaDa
             }
             // TODO: Assign the global checkpoint to the max_seqno of the safe commit if the index version >= 6.2
             final String translogUUID = Translog.createEmptyTranslog(
-                indexShard.shardPath().resolveTranslog(), SequenceNumbers.UNASSIGNED_SEQ_NO, shardId, indexShard.getPrimaryTerm());
+                indexShard.shardPath().resolveTranslog(), SequenceNumbers.UNASSIGNED_SEQ_NO, shardId,
+                indexShard.getPendingPrimaryTerm());
             store.associateIndexWithNewTranslog(translogUUID);
         } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) {
             // this is a fatal exception at this stage.
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTranslogOperationsRequest.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTranslogOperationsRequest.java
index 4649462692006..be399e0f81fd0 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTranslogOperationsRequest.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTranslogOperationsRequest.java
@@ -66,7 +66,7 @@ public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         recoveryId = in.readLong();
         shardId = ShardId.readShardId(in);
-        operations = Translog.readOperations(in);
+        operations = Translog.readOperations(in, "recovery");
         totalTranslogOps = in.readVInt();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
index 2bd842e72b107..aad55e12cefff 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
@@ -527,8 +527,7 @@ private static List<Object> appendValues(Object maybeList, Object value) {
 
     private static void appendValues(List<Object> list, Object value) {
         if (value instanceof List) {
-            List<?> valueList = (List<?>) value;
-            valueList.stream().forEach(list::add);
+            list.addAll((List<?>) value);
         } else {
             list.add(value);
         }
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java
index 46b11f7ac141d..01bc402e43ba5 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java
@@ -39,6 +39,9 @@
  * Holder class for several ingest related services.
  */
 public class IngestService {
+
+    public static final String NOOP_PIPELINE_NAME = "_none";
+
     private final PipelineStore pipelineStore;
     private final PipelineExecutionService pipelineExecutionService;
 
diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java b/server/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
index a8aca4fdfe59d..56d44ee888122 100644
--- a/server/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
+++ b/server/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterStateApplier;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
@@ -73,12 +72,16 @@ protected void doRun() throws Exception {
                         UpdateRequest updateRequest = (UpdateRequest) actionRequest;
                         indexRequest = updateRequest.docAsUpsert() ? updateRequest.doc() : updateRequest.upsertRequest();
                     }
-                    if (indexRequest != null && Strings.hasText(indexRequest.getPipeline())) {
+                    if (indexRequest == null) {
+                        continue;
+                    }
+                    String pipeline = indexRequest.getPipeline();
+                    if (IngestService.NOOP_PIPELINE_NAME.equals(pipeline) == false) {
                         try {
                             innerExecute(indexRequest, getPipeline(indexRequest.getPipeline()));
                             //this shouldn't be needed here but we do it for consistency with index api
                             // which requires it to prevent double execution
-                            indexRequest.setPipeline(null);
+                            indexRequest.setPipeline(IngestService.NOOP_PIPELINE_NAME);
                         } catch (Exception e) {
                             itemFailureHandler.accept(indexRequest, e);
                         }
diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/server/src/main/java/org/elasticsearch/ingest/PipelineStore.java
index c6dce0bd45b3c..9fceaf1a9a573 100644
--- a/server/src/main/java/org/elasticsearch/ingest/PipelineStore.java
+++ b/server/src/main/java/org/elasticsearch/ingest/PipelineStore.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.action.ingest.PutPipelineRequest;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
@@ -121,13 +121,13 @@ public String getType() {
     /**
      * Deletes the pipeline specified by id in the request.
      */
-    public void delete(ClusterService clusterService, DeletePipelineRequest request, ActionListener<WritePipelineResponse> listener) {
+    public void delete(ClusterService clusterService, DeletePipelineRequest request, ActionListener<AcknowledgedResponse> listener) {
         clusterService.submitStateUpdateTask("delete-pipeline-" + request.getId(),
-                new AckedClusterStateUpdateTask<WritePipelineResponse>(request, listener) {
+                new AckedClusterStateUpdateTask<AcknowledgedResponse>(request, listener) {
 
             @Override
-            protected WritePipelineResponse newResponse(boolean acknowledged) {
-                return new WritePipelineResponse(acknowledged);
+            protected AcknowledgedResponse newResponse(boolean acknowledged) {
+                return new AcknowledgedResponse(acknowledged);
             }
 
             @Override
@@ -169,15 +169,15 @@ ClusterState innerDelete(DeletePipelineRequest request, ClusterState currentStat
      * Stores the specified pipeline definition in the request.
      */
     public void put(ClusterService clusterService, Map<DiscoveryNode, IngestInfo> ingestInfos, PutPipelineRequest request,
-                    ActionListener<WritePipelineResponse> listener) throws Exception {
+                    ActionListener<AcknowledgedResponse> listener) throws Exception {
         // validates the pipeline and processor configuration before submitting a cluster update task:
         validatePipeline(ingestInfos, request);
         clusterService.submitStateUpdateTask("put-pipeline-" + request.getId(),
-                new AckedClusterStateUpdateTask<WritePipelineResponse>(request, listener) {
+                new AckedClusterStateUpdateTask<AcknowledgedResponse>(request, listener) {
 
             @Override
-            protected WritePipelineResponse newResponse(boolean acknowledged) {
-                return new WritePipelineResponse(acknowledged);
+            protected AcknowledgedResponse newResponse(boolean acknowledged) {
+                return new AcknowledgedResponse(acknowledged);
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index c1ce864223b70..c65488bd08eb7 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -231,7 +231,14 @@ public static final Settings addNodeNameIfNeeded(Settings settings, final String
     }
 
     private static final String CLIENT_TYPE = "node";
+
     private final Lifecycle lifecycle = new Lifecycle();
+
+    /**
+     * Logger initialized in the ctor because if it were initialized statically
+     * then it wouldn't get the node name.
+     */
+    private final Logger logger;
     private final Injector injector;
     private final Settings settings;
     private final Settings originalSettings;
@@ -257,13 +264,9 @@ public Node(Environment environment) {
     }
 
     protected Node(final Environment environment, Collection<Class<? extends Plugin>> classpathPlugins) {
+        logger = Loggers.getLogger(Node.class);
         final List<Closeable> resourcesToClose = new ArrayList<>(); // register everything we need to release in the case of an error
         boolean success = false;
-        {
-            // use temp logger just to say we are starting. we can't use it later on because the node name might not be set
-            Logger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(environment.settings()));
-            logger.info("initializing ...");
-        }
         try {
             originalSettings = environment.settings();
             Settings tmpSettings = Settings.builder().put(environment.settings())
@@ -279,7 +282,6 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
             final boolean hadPredefinedNodeName = NODE_NAME_SETTING.exists(tmpSettings);
             final String nodeId = nodeEnvironment.nodeId();
             tmpSettings = addNodeNameIfNeeded(tmpSettings, nodeId);
-            final Logger logger = Loggers.getLogger(Node.class, tmpSettings);
             // this must be captured after the node name is possibly added to the settings
             final String nodeName = NODE_NAME_SETTING.get(tmpSettings);
             if (hadPredefinedNodeName == false) {
@@ -347,8 +349,7 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
                 getCustomNameResolvers(pluginsService.filterPlugins(DiscoveryPlugin.class)));
 
             List<ClusterPlugin> clusterPlugins = pluginsService.filterPlugins(ClusterPlugin.class);
-            final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool,
-               ClusterModule.getClusterStateCustomSuppliers(clusterPlugins));
+            final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
             clusterService.addStateApplier(scriptModule.getScriptService());
             resourcesToClose.add(clusterService);
             final IngestService ingestService = new IngestService(settings, threadPool, this.environment,
@@ -631,7 +632,6 @@ public Node start() throws NodeValidationException {
             return this;
         }
 
-        Logger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(settings));
         logger.info("starting ...");
         pluginLifecycleComponents.forEach(LifecycleComponent::start);
 
@@ -742,7 +742,6 @@ private Node stop() {
         if (!lifecycle.moveToStopped()) {
             return this;
         }
-        Logger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(settings));
         logger.info("stopping ...");
 
         injector.getInstance(ResourceWatcherService.class).stop();
@@ -785,7 +784,6 @@ public synchronized void close() throws IOException {
             return;
         }
 
-        Logger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(settings));
         logger.info("closing ...");
         List<Closeable> toClose = new ArrayList<>();
         StopWatch stopWatch = new StopWatch("node_close");
diff --git a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java
index 61145c7a1d7cf..a1274b9346cc8 100644
--- a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java
@@ -24,7 +24,6 @@
 import java.util.Map;
 import java.util.function.Supplier;
 
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -66,12 +65,4 @@ default Map<String, Supplier<ShardsAllocator>> getShardsAllocators(Settings sett
     default void onNodeStarted() {
     }
 
-    /**
-     * Returns a map of {@link ClusterState.Custom} supplier that should be invoked to initialize the initial clusterstate.
-     * This allows custom clusterstate extensions to be always present and prevents invariants where clusterstates are published
-     * but customs are not initialized.
-     *
-     * TODO: Remove this whole concept of InitialClusterStateCustomSupplier, it's not used anymore
-     */
-    default Map<String, Supplier<ClusterState.Custom>> getInitialClusterStateCustomSupplier() { return Collections.emptyMap(); }
 }
diff --git a/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
index 952aa76fd17a9..b134db4451718 100644
--- a/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
@@ -48,6 +48,7 @@
 import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.search.rescore.Rescorer;
+import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
 
@@ -149,31 +150,61 @@ public ScoreFunctionSpec(String name, Writeable.Reader<T> reader, ScoreFunctionP
      * Specification for a {@link Suggester}.
      */
     class SuggesterSpec<T extends SuggestionBuilder<T>> extends SearchExtensionSpec<T, CheckedFunction<XContentParser, T, IOException>> {
+
+        private Writeable.Reader<? extends Suggest.Suggestion> suggestionReader;
+
         /**
          * Specification of custom {@link Suggester}.
          *
          * @param name holds the names by which this suggester might be parsed. The {@link ParseField#getPreferredName()} is special as it
-         *        is the name by under which the reader is registered. So it is the name that the query should use as its
-         *        {@link NamedWriteable#getWriteableName()} too.
-         * @param reader the reader registered for this suggester's builder. Typically a reference to a constructor that takes a
+         *        is the name by under which the request builder and Suggestion response readers are registered. So it is the name that the
+         *        query and Suggestion response should use as their {@link NamedWriteable#getWriteableName()} return values too.
+         * @param builderReader the reader registered for this suggester's builder. Typically a reference to a constructor that takes a
          *        {@link StreamInput}
-         * @param parser the parser the reads the query suggester from xcontent
+         * @param builderParser a parser that reads the suggester's builder from xcontent
+         * @param suggestionReader the reader registered for this suggester's Suggestion response. Typically a reference to a constructor
+         *        that takes a {@link StreamInput}
          */
-        public SuggesterSpec(ParseField name, Writeable.Reader<T> reader, CheckedFunction<XContentParser, T, IOException> parser) {
-            super(name, reader, parser);
+        public SuggesterSpec(
+                ParseField name,
+                Writeable.Reader<T> builderReader,
+                CheckedFunction<XContentParser, T, IOException> builderParser,
+                Writeable.Reader<? extends Suggest.Suggestion> suggestionReader) {
+
+            super(name, builderReader, builderParser);
+            setSuggestionReader(suggestionReader);
         }
 
         /**
          * Specification of custom {@link Suggester}.
          *
-         * @param name the name by which this suggester might be parsed or deserialized. Make sure that the query builder returns this name
-         *        for {@link NamedWriteable#getWriteableName()}.
-         * @param reader the reader registered for this suggester's builder. Typically a reference to a constructor that takes a
+         * @param name the name by which this suggester might be parsed or deserialized. Make sure that the query builder and Suggestion
+         *        response reader return this name for {@link NamedWriteable#getWriteableName()}.
+         * @param builderReader the reader registered for this suggester's builder. Typically a reference to a constructor that takes a
          *        {@link StreamInput}
-         * @param parser the parser the reads the suggester builder from xcontent
+         * @param builderParser a parser that reads the suggester's builder from xcontent
+         * @param suggestionReader the reader registered for this suggester's Suggestion response. Typically a reference to a constructor
+         *        that takes a {@link StreamInput}
          */
-        public SuggesterSpec(String name, Writeable.Reader<T> reader, CheckedFunction<XContentParser, T, IOException> parser) {
-            super(name, reader, parser);
+        public SuggesterSpec(
+                String name,
+                Writeable.Reader<T> builderReader,
+                CheckedFunction<XContentParser, T, IOException> builderParser,
+                Writeable.Reader<? extends Suggest.Suggestion> suggestionReader) {
+
+            super(name, builderReader, builderParser);
+            setSuggestionReader(suggestionReader);
+        }
+
+        private void setSuggestionReader(Writeable.Reader<? extends Suggest.Suggestion> reader) {
+            this.suggestionReader = reader;
+        }
+
+        /**
+         * Returns the reader used to read the {@link Suggest.Suggestion} generated by this suggester
+         */
+        public Writeable.Reader<? extends Suggest.Suggestion> getSuggestionReader() {
+            return this.suggestionReader;
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestMainAction.java b/server/src/main/java/org/elasticsearch/rest/action/RestMainAction.java
index 250ee209587e0..9efc8f526f3d5 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/RestMainAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/RestMainAction.java
@@ -67,4 +67,9 @@ static BytesRestResponse convertMainResponse(MainResponse response, RestRequest
         response.toXContent(builder, request);
         return new BytesRestResponse(RestStatus.OK, builder);
     }
+
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index 3a76c7ca0c952..1a859933ad3fe 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -380,7 +381,8 @@ Table buildTable(RestRequest request, Index[] indices, ClusterHealthResponse res
             table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getDeleted());
 
             table.addCell(indexMetaData.getCreationDate());
-            table.addCell(ZonedDateTime.ofInstant(Instant.ofEpochMilli(indexMetaData.getCreationDate()), ZoneOffset.UTC));
+            ZonedDateTime creationTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(indexMetaData.getCreationDate()), ZoneOffset.UTC);
+            table.addCell(DateFormatters.forPattern("strict_date_time").format(creationTime));
 
             table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size());
             table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size());
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index d8f8c59caf6d0..a03fa2c059eea 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.rest.action.RestActionListener;
 import org.elasticsearch.rest.action.RestResponseListener;
 
+import java.time.Instant;
 import java.util.Locale;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -230,7 +231,8 @@ private Table buildTable(RestRequest request, ClusterStateResponse state, Indice
 
             if (shard.unassignedInfo() != null) {
                 table.addCell(shard.unassignedInfo().getReason());
-                table.addCell(UnassignedInfo.DATE_TIME_FORMATTER.printer().print(shard.unassignedInfo().getUnassignedTimeInMillis()));
+                Instant unassignedTime = Instant.ofEpochMilli(shard.unassignedInfo().getUnassignedTimeInMillis());
+                table.addCell(UnassignedInfo.DATE_TIME_FORMATTER.format(unassignedTime));
                 table.addCell(TimeValue.timeValueMillis(System.currentTimeMillis() - shard.unassignedInfo().getUnassignedTimeInMillis()));
                 table.addCell(shard.unassignedInfo().getDetails());
             } else {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
index 6d44e9aa856bd..2da5e432ca32a 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
@@ -25,6 +25,8 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.CompoundDateTimeFormatter;
+import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -32,9 +34,9 @@
 import org.elasticsearch.rest.action.RestResponseListener;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotState;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
 
+import java.time.Instant;
+import java.time.ZoneOffset;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -97,7 +99,7 @@ protected Table getTableWithHeader(RestRequest request) {
                 .endHeaders();
     }
 
-    private DateTimeFormatter dateFormat = DateTimeFormat.forPattern("HH:mm:ss");
+    private static final CompoundDateTimeFormatter FORMATTER = DateFormatters.forPattern("HH:mm:ss").withZone(ZoneOffset.UTC);
 
     private Table buildTable(RestRequest req, GetSnapshotsResponse getSnapshotsResponse) {
         Table table = getTableWithHeader(req);
@@ -107,9 +109,9 @@ private Table buildTable(RestRequest req, GetSnapshotsResponse getSnapshotsRespo
             table.addCell(snapshotStatus.snapshotId().getName());
             table.addCell(snapshotStatus.state());
             table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.startTime(), TimeUnit.MILLISECONDS));
-            table.addCell(dateFormat.print(snapshotStatus.startTime()));
+            table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.startTime())));
             table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.endTime(), TimeUnit.MILLISECONDS));
-            table.addCell(dateFormat.print(snapshotStatus.endTime()));
+            table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.endTime())));
             final long durationMillis;
             if (snapshotStatus.state() == SnapshotState.IN_PROGRESS) {
                 durationMillis = System.currentTimeMillis() - snapshotStatus.startTime();
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
index c0ebddc2908d1..7d14422b37c2b 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
@@ -27,17 +27,20 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.CompoundDateTimeFormatter;
+import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestResponseListener;
 import org.elasticsearch.tasks.TaskInfo;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
 
+import java.time.Instant;
+import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -122,7 +125,7 @@ protected Table getTableWithHeader(final RestRequest request) {
         return table;
     }
 
-    private DateTimeFormatter dateFormat = DateTimeFormat.forPattern("HH:mm:ss");
+    private static final CompoundDateTimeFormatter FORMATTER = DateFormatters.forPattern("HH:mm:ss").withZone(ZoneOffset.UTC);
 
     private void buildRow(Table table, boolean fullId, boolean detailed, DiscoveryNodes discoveryNodes, TaskInfo taskInfo) {
         table.startRow();
@@ -139,7 +142,7 @@ private void buildRow(Table table, boolean fullId, boolean detailed, DiscoveryNo
         }
         table.addCell(taskInfo.getType());
         table.addCell(taskInfo.getStartTime());
-        table.addCell(dateFormat.print(taskInfo.getStartTime()));
+        table.addCell(FORMATTER.format(Instant.ofEpochMilli(taskInfo.getStartTime())));
         table.addCell(taskInfo.getRunningTimeNanos());
         table.addCell(TimeValue.timeValueNanos(taskInfo.getRunningTimeNanos()).toString());
 
@@ -159,7 +162,7 @@ private void buildRow(Table table, boolean fullId, boolean detailed, DiscoveryNo
     private void buildGroups(Table table, boolean fullId, boolean detailed, List<TaskGroup> taskGroups) {
         DiscoveryNodes discoveryNodes = nodesInCluster.get();
         List<TaskGroup> sortedGroups = new ArrayList<>(taskGroups);
-        sortedGroups.sort((o1, o2) -> Long.compare(o1.getTaskInfo().getStartTime(), o2.getTaskInfo().getStartTime()));
+        sortedGroups.sort(Comparator.comparingLong(o -> o.getTaskInfo().getStartTime()));
         for (TaskGroup taskGroup : sortedGroups) {
             buildRow(table, fullId, detailed, discoveryNodes, taskGroup.getTaskInfo());
             buildGroups(table, fullId, detailed, taskGroup.getChildTasks());
diff --git a/server/src/main/java/org/elasticsearch/script/BucketAggregationScript.java b/server/src/main/java/org/elasticsearch/script/BucketAggregationScript.java
new file mode 100644
index 0000000000000..5fa8d1fbf94a4
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/script/BucketAggregationScript.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import java.util.Map;
+
+/**
+ * A script used in bucket aggregations that returns a {@code double} value.
+ */
+public abstract class BucketAggregationScript {
+
+    public static final String[] PARAMETERS = {};
+
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("bucket_aggregation", Factory.class);
+
+    /**
+     * The generic runtime parameters for the script.
+     */
+    private final Map<String, Object> params;
+
+    public BucketAggregationScript(Map<String, Object> params) {
+        this.params = params;
+    }
+
+    /**
+     * Return the parameters for this script.
+     */
+    public Map<String, Object> getParams() {
+        return params;
+    }
+
+    public abstract Double execute();
+
+    public interface Factory {
+        BucketAggregationScript newInstance(Map<String, Object> params);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/script/BucketAggregationSelectorScript.java b/server/src/main/java/org/elasticsearch/script/BucketAggregationSelectorScript.java
new file mode 100644
index 0000000000000..a8e2fad7cdcda
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/script/BucketAggregationSelectorScript.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import java.util.Map;
+
+/**
+ * A script used in bucket aggregations that returns a {@code boolean} value.
+ */
+public abstract class BucketAggregationSelectorScript {
+
+    public static final String[] PARAMETERS = {};
+
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("aggregation_selector", Factory.class);
+
+    /**
+     * The generic runtime parameters for the script.
+     */
+    private final Map<String, Object> params;
+
+    public BucketAggregationSelectorScript(Map<String, Object> params) {
+        this.params = params;
+    }
+
+    /**
+     * Return the parameters for this script.
+     */
+    public Map<String, Object> getParams() {
+        return params;
+    }
+
+    public abstract boolean execute();
+
+    public interface Factory {
+        BucketAggregationSelectorScript newInstance(Map<String, Object> params);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/script/ExecutableScript.java b/server/src/main/java/org/elasticsearch/script/ExecutableScript.java
index 2f7a01c37980d..d0d8020371bd1 100644
--- a/server/src/main/java/org/elasticsearch/script/ExecutableScript.java
+++ b/server/src/main/java/org/elasticsearch/script/ExecutableScript.java
@@ -46,8 +46,4 @@ interface Factory {
     }
 
     ScriptContext<Factory> CONTEXT = new ScriptContext<>("executable", Factory.class);
-
-    // TODO: remove these once each has its own script interface
-    ScriptContext<Factory> AGGS_CONTEXT = new ScriptContext<>("aggs_executable", Factory.class);
-    ScriptContext<Factory> UPDATE_CONTEXT = new ScriptContext<>("update", Factory.class);
 }
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptModule.java b/server/src/main/java/org/elasticsearch/script/ScriptModule.java
index a3da1dafe48d7..f04e690fa425c 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptModule.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptModule.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.script;
 
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.search.aggregations.pipeline.movfn.MovingFunctionScript;
+
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -27,12 +32,6 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.search.aggregations.pipeline.movfn.MovingFunctionScript;
-
-
 /**
  * Manages building {@link ScriptService}.
  */
@@ -47,8 +46,10 @@ public class ScriptModule {
             SearchScript.SCRIPT_SORT_CONTEXT,
             SearchScript.TERMS_SET_QUERY_CONTEXT,
             ExecutableScript.CONTEXT,
-            ExecutableScript.AGGS_CONTEXT,
-            ExecutableScript.UPDATE_CONTEXT,
+            UpdateScript.CONTEXT,
+            BucketAggregationScript.CONTEXT,
+            BucketAggregationSelectorScript.CONTEXT,
+            SignificantTermsHeuristicScoreScript.CONTEXT,
             IngestScript.CONTEXT,
             FilterScript.CONTEXT,
             SimilarityScript.CONTEXT,
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java
index ca79e3b80fc81..9768547b89819 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -285,7 +285,7 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
         // TODO: fix this through some API or something, that's wrong
         // special exception to prevent expressions from compiling as update or mapping scripts
         boolean expression = "expression".equals(lang);
-        boolean notSupported = context.name.equals(ExecutableScript.UPDATE_CONTEXT.name);
+        boolean notSupported = context.name.equals(UpdateScript.CONTEXT.name);
         if (expression && notSupported) {
             throw new UnsupportedOperationException("scripts of type [" + script.getType() + "]," +
                 " operation [" + context.name + "] and lang [" + lang + "] are not supported");
diff --git a/server/src/main/java/org/elasticsearch/script/SignificantTermsHeuristicScoreScript.java b/server/src/main/java/org/elasticsearch/script/SignificantTermsHeuristicScoreScript.java
new file mode 100644
index 0000000000000..0296bc36ce107
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/script/SignificantTermsHeuristicScoreScript.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import java.util.Map;
+
+/**
+ * A script used in significant terms heuristic scoring.
+ */
+public abstract class SignificantTermsHeuristicScoreScript {
+
+    public static final String[] PARAMETERS = { "params" };
+
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("script_heuristic", Factory.class);
+
+    public abstract double execute(Map<String, Object> params);
+
+    public interface Factory {
+        SignificantTermsHeuristicScoreScript newInstance();
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/script/UpdateScript.java b/server/src/main/java/org/elasticsearch/script/UpdateScript.java
new file mode 100644
index 0000000000000..c6a1d5dd9eabb
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/script/UpdateScript.java
@@ -0,0 +1,52 @@
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import java.util.Map;
+
+/**
+ * An update script.
+ */
+public abstract class UpdateScript {
+
+    public static final String[] PARAMETERS = { "ctx" };
+
+    /** The context used to compile {@link UpdateScript} factories. */
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("update", Factory.class);
+
+    /** The generic runtime parameters for the script. */
+    private final Map<String, Object> params;
+
+    public UpdateScript(Map<String, Object> params) {
+        this.params = params;
+    }
+
+    /** Return the parameters for this script. */
+    public Map<String, Object> getParams() {
+        return params;
+    }
+
+    public abstract void execute(Map<String, Object> ctx);
+
+    public interface Factory {
+        UpdateScript newInstance(Map<String, Object> params);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java
index 8c688cbf4466a..28a600c0d21ef 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchHit.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java
@@ -19,6 +19,16 @@
 
 package org.elasticsearch.search;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
 import org.apache.lucene.search.Explanation;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.OriginalIndices;
@@ -51,16 +61,6 @@
 import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.transport.RemoteClusterAware;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static java.util.Collections.unmodifiableMap;
@@ -107,6 +107,9 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable<D
     @Nullable
     private SearchShardTarget shard;
 
+    //These two fields normally get set when setting the shard target, so they hold the same values as the target thus don't get
+    //serialized over the wire. When parsing hits back from xcontent though, in most of the cases (whenever explanation is disabled)
+    //we can't rebuild the shard target object so we need to set these manually for users retrieval.
     private transient String index;
     private transient String clusterAlias;
 
@@ -551,7 +554,26 @@ public static SearchHit createFromMap(Map<String, Object> values) {
         Map<String, DocumentField> fields = get(Fields.FIELDS, values, Collections.emptyMap());
 
         SearchHit searchHit = new SearchHit(-1, id, type, nestedIdentity, fields);
-        searchHit.index = get(Fields._INDEX, values, null);
+        String index = get(Fields._INDEX, values, null);
+        String clusterAlias = null;
+        if (index != null) {
+            int indexOf = index.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR);
+            if (indexOf > 0) {
+                clusterAlias = index.substring(0, indexOf);
+                index = index.substring(indexOf + 1);
+            }
+        }
+        ShardId shardId = get(Fields._SHARD, values, null);
+        String nodeId = get(Fields._NODE, values, null);
+        if (shardId != null && nodeId != null) {
+            assert shardId.getIndexName().equals(index);
+            searchHit.shard(new SearchShardTarget(nodeId, shardId, clusterAlias, OriginalIndices.NONE));
+        } else {
+            //these fields get set anyways when setting the shard target,
+            //but we set them explicitly when we don't have enough info to rebuild the shard target
+            searchHit.index = index;
+            searchHit.clusterAlias = clusterAlias;
+        }
         searchHit.score(get(Fields._SCORE, values, DEFAULT_SCORE));
         searchHit.version(get(Fields._VERSION, values, -1L));
         searchHit.sortValues(get(Fields.SORT, values, SearchSortValues.EMPTY));
@@ -561,12 +583,7 @@ public static SearchHit createFromMap(Map<String, Object> values) {
         searchHit.setInnerHits(get(Fields.INNER_HITS, values, null));
         List<String> matchedQueries = get(Fields.MATCHED_QUERIES, values, null);
         if (matchedQueries != null) {
-            searchHit.matchedQueries(matchedQueries.toArray(new String[matchedQueries.size()]));
-        }
-        ShardId shardId = get(Fields._SHARD, values, null);
-        String nodeId = get(Fields._NODE, values, null);
-        if (shardId != null && nodeId != null) {
-            searchHit.shard(new SearchShardTarget(nodeId, shardId, null, OriginalIndices.NONE));
+            searchHit.matchedQueries(matchedQueries.toArray(new String[0]));
         }
         return searchHit;
     }
@@ -602,16 +619,24 @@ private static void declareMetaDataFields(ObjectParser<Map<String, Object>, Void
         for (String metadatafield : MapperService.getAllMetaFields()) {
             if (metadatafield.equals(Fields._ID) == false && metadatafield.equals(Fields._INDEX) == false
                     && metadatafield.equals(Fields._TYPE) == false) {
-                parser.declareField((map, field) -> {
-                    @SuppressWarnings("unchecked")
-                    Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
-                            v -> new HashMap<String, DocumentField>());
-                    fieldMap.put(field.getName(), field);
-                }, (p, c) -> {
-                    List<Object> values = new ArrayList<>();
-                    values.add(parseFieldsValue(p));
-                    return new DocumentField(metadatafield, values);
-                }, new ParseField(metadatafield), ValueType.VALUE);
+                if (metadatafield.equals(IgnoredFieldMapper.NAME)) {
+                    parser.declareObjectArray((map, list) -> {
+                            @SuppressWarnings("unchecked")
+                            Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
+                                v -> new HashMap<String, DocumentField>());
+                            DocumentField field = new DocumentField(metadatafield, list);
+                            fieldMap.put(field.getName(), field);
+                        }, (p, c) -> parseFieldsValue(p),
+                        new ParseField(metadatafield));
+                } else {
+                    parser.declareField((map, field) -> {
+                            @SuppressWarnings("unchecked")
+                            Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
+                                v -> new HashMap<String, DocumentField>());
+                            fieldMap.put(field.getName(), field);
+                        }, (p, c) -> new DocumentField(metadatafield, Collections.singletonList(parseFieldsValue(p))),
+                        new ParseField(metadatafield), ValueType.VALUE);
+                }
             }
         }
     }
@@ -834,13 +859,15 @@ public boolean equals(Object obj) {
                 && Arrays.equals(matchedQueries, other.matchedQueries)
                 && Objects.equals(explanation, other.explanation)
                 && Objects.equals(shard, other.shard)
-                && Objects.equals(innerHits, other.innerHits);
+                && Objects.equals(innerHits, other.innerHits)
+                && Objects.equals(index, other.index)
+                && Objects.equals(clusterAlias, other.clusterAlias);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(id, type, nestedIdentity, version, source, fields, getHighlightFields(), Arrays.hashCode(matchedQueries),
-                explanation, shard, innerHits);
+                explanation, shard, innerHits, index, clusterAlias);
     }
 
     /**
@@ -958,4 +985,9 @@ public int hashCode() {
             return Objects.hash(field, offset, child);
         }
     }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index 99b47cf83e2ac..e2baad5d60611 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -247,13 +247,17 @@
 import org.elasticsearch.search.sort.ScoreSortBuilder;
 import org.elasticsearch.search.sort.ScriptSortBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
 import org.elasticsearch.search.suggest.phrase.Laplace;
 import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
 import org.elasticsearch.search.suggest.phrase.SmoothingModel;
 import org.elasticsearch.search.suggest.phrase.StupidBackoff;
+import org.elasticsearch.search.suggest.term.TermSuggestion;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 
 import java.util.ArrayList;
@@ -590,9 +594,14 @@ public static void registerSmoothingModels(List<Entry> namedWriteables) {
     private void registerSuggesters(List<SearchPlugin> plugins) {
         registerSmoothingModels(namedWriteables);
 
-        registerSuggester(new SuggesterSpec<>("term", TermSuggestionBuilder::new, TermSuggestionBuilder::fromXContent));
-        registerSuggester(new SuggesterSpec<>("phrase", PhraseSuggestionBuilder::new, PhraseSuggestionBuilder::fromXContent));
-        registerSuggester(new SuggesterSpec<>("completion", CompletionSuggestionBuilder::new, CompletionSuggestionBuilder::fromXContent));
+        registerSuggester(new SuggesterSpec<>(TermSuggestionBuilder.SUGGESTION_NAME,
+            TermSuggestionBuilder::new, TermSuggestionBuilder::fromXContent, TermSuggestion::new));
+
+        registerSuggester(new SuggesterSpec<>(PhraseSuggestionBuilder.SUGGESTION_NAME,
+            PhraseSuggestionBuilder::new, PhraseSuggestionBuilder::fromXContent, PhraseSuggestion::new));
+
+        registerSuggester(new SuggesterSpec<>(CompletionSuggestionBuilder.SUGGESTION_NAME,
+            CompletionSuggestionBuilder::new, CompletionSuggestionBuilder::fromXContent, CompletionSuggestion::new));
 
         registerFromPlugin(plugins, SearchPlugin::getSuggesters, this::registerSuggester);
     }
@@ -602,6 +611,10 @@ private void registerSuggester(SuggesterSpec<?> suggester) {
                 SuggestionBuilder.class, suggester.getName().getPreferredName(), suggester.getReader()));
         namedXContents.add(new NamedXContentRegistry.Entry(SuggestionBuilder.class, suggester.getName(),
                 suggester.getParser()));
+
+        namedWriteables.add(new NamedWriteableRegistry.Entry(
+            Suggest.Suggestion.class, suggester.getName().getPreferredName(), suggester.getSuggestionReader()
+        ));
     }
 
     private Map<String, Highlighter> setupHighlighters(Settings settings, List<SearchPlugin> plugins) {
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 59af043e0cf7a..4bf5e03b8a7cc 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -806,7 +806,7 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
                 throw new SearchContextException(context, "failed to create SearchContextHighlighter", e);
             }
         }
-        if (source.scriptFields() != null) {
+        if (source.scriptFields() != null && source.size() != 0) {
             int maxAllowedScriptFields = context.mapperService().getIndexSettings().getMaxScriptFields();
             if (source.scriptFields().size() > maxAllowedScriptFields) {
                 throw new IllegalArgumentException(
diff --git a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java
index 4ac742ff5d9fe..19c0f8c64d58b 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java
@@ -39,7 +39,7 @@ public final class SearchShardTarget implements Writeable, Comparable<SearchShar
 
     private final Text nodeId;
     private final ShardId shardId;
-    //original indices and cluster alias are only needed in the coordinating node throughout the search request execution.
+    //original indices are only needed in the coordinating node throughout the search request execution.
     //no need to serialize them as part of SearchShardTarget.
     private final transient OriginalIndices originalIndices;
     private final String clusterAlias;
@@ -96,6 +96,13 @@ public String getClusterAlias() {
         return clusterAlias;
     }
 
+    /**
+     * Returns the fully qualified index name, including the cluster alias.
+     */
+    public String getFullyQualifiedIndexName() {
+        return RemoteClusterAware.buildRemoteIndexName(getClusterAlias(), getIndex());
+    }
+
     @Override
     public int compareTo(SearchShardTarget o) {
         int i = shardId.getIndexName().compareTo(o.getIndex());
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
index 366060835d891..50a0c85c041c8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.internal.SearchContext;
+import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -53,7 +54,7 @@ public class AutoDateHistogramAggregationBuilder
 
     public static final String NAME = "auto_date_histogram";
 
-    public static final ParseField NUM_BUCKETS_FIELD = new ParseField("buckets");
+    private static final ParseField NUM_BUCKETS_FIELD = new ParseField("buckets");
 
     private static final ObjectParser<AutoDateHistogramAggregationBuilder, Void> PARSER;
     static {
@@ -63,6 +64,29 @@ public class AutoDateHistogramAggregationBuilder
         PARSER.declareInt(AutoDateHistogramAggregationBuilder::setNumBuckets, NUM_BUCKETS_FIELD);
     }
 
+    /**
+     *
+     * Build roundings, computed dynamically as roundings are time zone dependent.
+     * The current implementation probably should not be invoked in a tight loop.
+     * @return Array of RoundingInfo
+     */
+    static RoundingInfo[] buildRoundings(DateTimeZone timeZone) {
+        RoundingInfo[] roundings = new RoundingInfo[6];
+        roundings[0] = new RoundingInfo(createRounding(DateTimeUnit.SECOND_OF_MINUTE, timeZone),
+            1000L, 1, 5, 10, 30);
+        roundings[1] = new RoundingInfo(createRounding(DateTimeUnit.MINUTES_OF_HOUR, timeZone),
+            60 * 1000L, 1, 5, 10, 30);
+        roundings[2] = new RoundingInfo(createRounding(DateTimeUnit.HOUR_OF_DAY, timeZone),
+            60 * 60 * 1000L, 1, 3, 12);
+        roundings[3] = new RoundingInfo(createRounding(DateTimeUnit.DAY_OF_MONTH, timeZone),
+            24 * 60 * 60 * 1000L, 1, 7);
+        roundings[4] = new RoundingInfo(createRounding(DateTimeUnit.MONTH_OF_YEAR, timeZone),
+            30 * 24 * 60 * 60 * 1000L, 1, 3);
+        roundings[5] = new RoundingInfo(createRounding(DateTimeUnit.YEAR_OF_CENTURY, timeZone),
+            365 * 24 * 60 * 60 * 1000L, 1, 5, 10, 20, 50, 100);
+        return roundings;
+    }
+
     public static AutoDateHistogramAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
         return PARSER.parse(parser, new AutoDateHistogramAggregationBuilder(aggregationName), null);
     }
@@ -116,14 +140,7 @@ public int getNumBuckets() {
     @Override
     protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        RoundingInfo[] roundings = new RoundingInfo[6];
-        roundings[0] = new RoundingInfo(createRounding(DateTimeUnit.SECOND_OF_MINUTE), 1000L, 1, 5, 10, 30);
-        roundings[1] = new RoundingInfo(createRounding(DateTimeUnit.MINUTES_OF_HOUR), 60 * 1000L, 1, 5, 10, 30);
-        roundings[2] = new RoundingInfo(createRounding(DateTimeUnit.HOUR_OF_DAY), 60 * 60 * 1000L, 1, 3, 12);
-        roundings[3] = new RoundingInfo(createRounding(DateTimeUnit.DAY_OF_MONTH), 24 * 60 * 60 * 1000L, 1, 7);
-        roundings[4] = new RoundingInfo(createRounding(DateTimeUnit.MONTH_OF_YEAR), 30 * 24 * 60 * 60 * 1000L, 1, 3);
-        roundings[5] = new RoundingInfo(createRounding(DateTimeUnit.YEAR_OF_CENTURY), 365 * 24 * 60 * 60 * 1000L, 1, 5, 10, 20, 50, 100);
-
+        RoundingInfo[] roundings = buildRoundings(timeZone());
         int maxRoundingInterval = Arrays.stream(roundings,0, roundings.length-1)
             .map(rounding -> rounding.innerIntervals)
             .flatMapToInt(Arrays::stream)
@@ -139,10 +156,10 @@ public int getNumBuckets() {
         return new AutoDateHistogramAggregatorFactory(name, config, numBuckets, roundings, context, parent, subFactoriesBuilder, metaData);
     }
 
-    private Rounding createRounding(DateTimeUnit interval) {
+    private static Rounding createRounding(DateTimeUnit interval, DateTimeZone timeZone) {
         Rounding.Builder tzRoundingBuilder = Rounding.builder(interval);
-        if (timeZone() != null) {
-            tzRoundingBuilder.timeZone(timeZone());
+        if (timeZone != null) {
+            tzRoundingBuilder.timeZone(timeZone);
         }
         Rounding rounding = tzRoundingBuilder.build();
         return rounding;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
index 27c195cbdae75..6a78ca6724988 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
@@ -447,7 +447,8 @@ private BucketReduceResult addEmptyBuckets(BucketReduceResult currentResult, Red
         return new BucketReduceResult(list, roundingInfo, roundingIdx);
     }
 
-    private int getAppropriateRounding(long minKey, long maxKey, int roundingIdx, RoundingInfo[] roundings) {
+    private int getAppropriateRounding(long minKey, long maxKey, int roundingIdx,
+                                              RoundingInfo[] roundings) {
         if (roundingIdx == roundings.length - 1) {
             return roundingIdx;
         }
@@ -509,7 +510,8 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
                 pipelineAggregators(), getMetaData());
     }
 
-    private BucketReduceResult maybeMergeConsecutiveBuckets(BucketReduceResult reducedBucketsResult, ReduceContext reduceContext) {
+    private BucketReduceResult maybeMergeConsecutiveBuckets(BucketReduceResult reducedBucketsResult,
+                                                                   ReduceContext reduceContext) {
         List<Bucket> buckets = reducedBucketsResult.buckets;
         RoundingInfo roundingInfo = reducedBucketsResult.roundingInfo;
         int roundingIdx = reducedBucketsResult.roundingIdx;
@@ -539,7 +541,7 @@ private BucketReduceResult mergeConsecutiveBuckets(List<Bucket> reducedBuckets,
                 key = roundingInfo.rounding.round(bucket.key);
             }
             reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(bucket) - 1);
-            sameKeyedBuckets.add(createBucket(key, bucket.docCount, bucket.aggregations));
+            sameKeyedBuckets.add(new Bucket(Math.round(key), bucket.docCount, format, bucket.aggregations));
         }
         if (sameKeyedBuckets.isEmpty() == false) {
             reduceContext.consumeBucketsAndMaybeBreak(1);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantTerms.java
index 1b4739c184d5e..26c4ec420d080 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantTerms.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantTerms.java
@@ -175,7 +175,7 @@ static <B extends ParsedBucket> B parseSignificantTermsBucketXContent(final XCon
                         bucket.subsetDf = value;
                         bucket.setDocCount(value);
                     } else if (InternalSignificantTerms.SCORE.equals(currentFieldName)) {
-                        bucket.score = parser.longValue();
+                        bucket.score = parser.doubleValue();
                     } else if (InternalSignificantTerms.BG_COUNT.equals(currentFieldName)) {
                         bucket.supersetDf = parser.longValue();
                     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
index a2cbd49693a27..05415cf7d19a8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
@@ -28,12 +28,14 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.SignificantTermsHeuristicScoreScript;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
 
 public class ScriptHeuristic extends SignificanceHeuristic {
@@ -48,19 +50,21 @@ static class ExecutableScriptHeuristic extends ScriptHeuristic {
         private final LongAccessor supersetSizeHolder;
         private final LongAccessor subsetDfHolder;
         private final LongAccessor supersetDfHolder;
-        private final ExecutableScript executableScript;
+        private final SignificantTermsHeuristicScoreScript executableScript;
+        private final Map<String, Object> params = new HashMap<>();
 
-        ExecutableScriptHeuristic(Script script, ExecutableScript executableScript){
+        ExecutableScriptHeuristic(Script script, SignificantTermsHeuristicScoreScript executableScript) {
             super(script);
             subsetSizeHolder = new LongAccessor();
             supersetSizeHolder = new LongAccessor();
             subsetDfHolder = new LongAccessor();
             supersetDfHolder = new LongAccessor();
             this.executableScript = executableScript;
-            executableScript.setNextVar("_subset_freq", subsetDfHolder);
-            executableScript.setNextVar("_subset_size", subsetSizeHolder);
-            executableScript.setNextVar("_superset_freq", supersetDfHolder);
-            executableScript.setNextVar("_superset_size", supersetSizeHolder);
+            params.putAll(script.getParams());
+            params.put("_subset_freq", subsetDfHolder);
+            params.put("_subset_size", subsetSizeHolder);
+            params.put("_superset_freq", supersetDfHolder);
+            params.put("_superset_size", supersetSizeHolder);
         }
 
         @Override
@@ -69,7 +73,7 @@ public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long
             supersetSizeHolder.value = supersetSize;
             subsetDfHolder.value = subsetFreq;
             supersetDfHolder.value = supersetFreq;
-            return ((Number) executableScript.run()).doubleValue();
+            return executableScript.execute(params);
        }
     }
 
@@ -91,15 +95,15 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public SignificanceHeuristic rewrite(InternalAggregation.ReduceContext context) {
-        ExecutableScript.Factory factory = context.scriptService().compile(script, ExecutableScript.AGGS_CONTEXT);
-        return new ExecutableScriptHeuristic(script, factory.newInstance(script.getParams()));
+        SignificantTermsHeuristicScoreScript.Factory factory = context.scriptService().compile(script, SignificantTermsHeuristicScoreScript.CONTEXT);
+        return new ExecutableScriptHeuristic(script, factory.newInstance());
     }
 
     @Override
     public SignificanceHeuristic rewrite(SearchContext context) {
         QueryShardContext shardContext = context.getQueryShardContext();
-        ExecutableScript.Factory compiledScript = shardContext.getScriptService().compile(script, ExecutableScript.AGGS_CONTEXT);
-        return new ExecutableScriptHeuristic(script, compiledScript.newInstance(script.getParams()));
+        SignificantTermsHeuristicScoreScript.Factory compiledScript = shardContext.getScriptService().compile(script, SignificantTermsHeuristicScoreScript.CONTEXT);
+        return new ExecutableScriptHeuristic(script, compiledScript.newInstance());
     }
 
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
index 42337fbce0f98..042a30695c61d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
@@ -21,10 +21,9 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.BucketAggregationScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregations;
@@ -89,7 +88,8 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext
                 (InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket>) aggregation;
         List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = originalAgg.getBuckets();
 
-        ExecutableScript.Factory factory = reduceContext.scriptService().compile(script, ExecutableScript.AGGS_CONTEXT);
+        BucketAggregationScript.Factory factory =
+            reduceContext.scriptService().compile(script, BucketAggregationScript.CONTEXT);
         List<InternalMultiBucketAggregation.InternalBucket> newBuckets = new ArrayList<>();
         for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
             Map<String, Object> vars = new HashMap<>();
@@ -110,22 +110,15 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext
             if (skipBucket) {
                 newBuckets.add(bucket);
             } else {
-                ExecutableScript executableScript = factory.newInstance(vars);
-                Object returned = executableScript.run();
-                // no need to check for self references since only numbers are valid
+                Double returned = factory.newInstance(vars).execute();
                 if (returned == null) {
                     newBuckets.add(bucket);
                 } else {
-                    if ((returned instanceof Number) == false) {
-                        throw new AggregationExecutionException("series_arithmetic script for reducer [" + name()
-                                + "] must return a Number");
-                    }
                     final List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map(
-                            (p) -> (InternalAggregation) p).collect(Collectors.toList());
-                    aggs.add(new InternalSimpleValue(name(), ((Number) returned).doubleValue(), formatter,
-                            new ArrayList<>(), metaData()));
+                        (p) -> (InternalAggregation) p).collect(Collectors.toList());
+                    aggs.add(new InternalSimpleValue(name(), returned, formatter, new ArrayList<>(), metaData()));
                     InternalMultiBucketAggregation.InternalBucket newBucket = originalAgg.createBucket(new InternalAggregations(aggs),
-                            bucket);
+                        bucket);
                     newBuckets.add(newBucket);
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
index a54ad0ec21f39..06beab04aa605 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
@@ -22,7 +22,7 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.BucketAggregationSelectorScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
@@ -82,7 +82,8 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext
                 (InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket>) aggregation;
         List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = originalAgg.getBuckets();
 
-        ExecutableScript.Factory factory = reduceContext.scriptService().compile(script, ExecutableScript.AGGS_CONTEXT);
+        BucketAggregationSelectorScript.Factory factory =
+            reduceContext.scriptService().compile(script, BucketAggregationSelectorScript.CONTEXT);
         List<InternalMultiBucketAggregation.InternalBucket> newBuckets = new ArrayList<>();
         for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
             Map<String, Object> vars = new HashMap<>();
@@ -96,17 +97,8 @@ public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext
                 vars.put(varName, value);
             }
             // TODO: can we use one instance of the script for all buckets? it should be stateless?
-            ExecutableScript executableScript = factory.newInstance(vars);
-            Object scriptReturnValue = executableScript.run();
-            final boolean keepBucket;
-            // TODO: WTF!!!!!
-            if ("expression".equals(script.getLang())) {
-                double scriptDoubleValue = (double) scriptReturnValue;
-                keepBucket = scriptDoubleValue == 1.0;
-            } else {
-                keepBucket = (boolean) scriptReturnValue;
-            }
-            if (keepBucket) {
+            BucketAggregationSelectorScript executableScript = factory.newInstance(vars);
+            if (executableScript.execute()) {
                 newBuckets.add(bucket);
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketsort/BucketSortPipelineAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketsort/BucketSortPipelineAggregator.java
index de5e2638c6cea..e10d5c35800fe 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketsort/BucketSortPipelineAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketsort/BucketSortPipelineAggregator.java
@@ -180,7 +180,7 @@ public int compareTo(ComparableBucket that) {
     private static class TopNPriorityQueue extends PriorityQueue<ComparableBucket> {
 
         private TopNPriorityQueue(int n) {
-            super(n, false);
+            super(n);
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationPath.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationPath.java
index 1a5539cab1816..ac97c48882ac1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationPath.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationPath.java
@@ -288,11 +288,14 @@ public Aggregator resolveTopmostAggregator(Aggregator root) {
     public void validate(Aggregator root) throws AggregationExecutionException {
         Aggregator aggregator = root;
         for (int i = 0; i < pathElements.size(); i++) {
-            aggregator = ProfilingAggregator.unwrap(aggregator.subAggregator(pathElements.get(i).name));
+            String name = pathElements.get(i).name;
+            aggregator = ProfilingAggregator.unwrap(aggregator.subAggregator(name));
             if (aggregator == null) {
-                throw new AggregationExecutionException("Invalid aggregator order path [" + this + "]. Unknown aggregation ["
-                        + pathElements.get(i).name + "]");
+                throw new AggregationExecutionException("Invalid aggregator order path [" + this + "]. The " +
+                    "provided aggregation [" + name + "] either does not exist, or is a pipeline aggregation " +
+                    "and cannot be used to sort the buckets.");
             }
+
             if (i < pathElements.size() - 1) {
 
                 // we're in the middle of the path, so the aggregator can only be a single-bucket aggregator
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java
index ac3c8f682ba62..1227efb5ea0af 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java
@@ -27,6 +27,7 @@
 
 import java.io.IOException;
 import java.lang.reflect.Array;
+import java.time.ZonedDateTime;
 import java.util.Collection;
 
 /**
@@ -54,6 +55,9 @@ public boolean advanceExact(int target) throws IOException {
         } else if (value instanceof ReadableInstant) {
             resize(1);
             values[0] = ((ReadableInstant) value).getMillis();
+        } else if (value instanceof ZonedDateTime) {
+            resize(1);
+            values[0] = ((ZonedDateTime) value).toInstant().toEpochMilli();
         } else if (value.getClass().isArray()) {
             int length = Array.getLength(value);
             if (length == 0) {
@@ -89,6 +93,8 @@ private static double toDoubleValue(Object o) {
         } else if (o instanceof ReadableInstant) {
             // Dates are exposed in scripts as ReadableDateTimes but aggregations want them to be numeric
             return ((ReadableInstant) o).getMillis();
+        } else if (o instanceof ZonedDateTime) {
+            return ((ZonedDateTime) o).toInstant().toEpochMilli();
         } else if (o instanceof Boolean) {
             // We do expose boolean fields as boolean in scripts, however aggregations still expect
             // that scripts return the same internal representation as regular fields, so boolean
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java
index 818a9d9fd8ddf..cdc448bd04130 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java
@@ -28,6 +28,7 @@
 
 import java.io.IOException;
 import java.lang.reflect.Array;
+import java.time.ZonedDateTime;
 import java.util.Collection;
 import java.util.Iterator;
 
@@ -91,6 +92,8 @@ private static long toLongValue(Object o) {
         } else if (o instanceof ReadableInstant) {
             // Dates are exposed in scripts as ReadableDateTimes but aggregations want them to be numeric
             return ((ReadableInstant) o).getMillis();
+        } else if (o instanceof ZonedDateTime) {
+            return ((ZonedDateTime) o).toInstant().toEpochMilli();
         } else if (o instanceof Boolean) {
             // We do expose boolean fields as boolean in scripts, however aggregations still expect
             // that scripts return the same internal representation as regular fields, so boolean
diff --git a/server/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java b/server/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java
index 90e35c34e28f8..ccab5e2cb93b3 100644
--- a/server/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java
@@ -247,6 +247,6 @@ public CollapseContext build(SearchContext context) {
                 + field + "`, " + "only indexed field can retrieve `inner_hits`");
         }
 
-        return new CollapseContext(fieldType, innerHits);
+        return new CollapseContext(field, fieldType, innerHits);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java b/server/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java
index 95fee901a30e1..82a7657f18079 100644
--- a/server/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java
+++ b/server/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java
@@ -25,24 +25,29 @@
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.query.InnerHitBuilder;
 
-import java.util.Collections;
 import java.util.List;
 
 /**
  * Context used for field collapsing
  */
 public class CollapseContext {
+    private final String fieldName;
     private final MappedFieldType fieldType;
     private final List<InnerHitBuilder> innerHits;
 
-    public CollapseContext(MappedFieldType fieldType, InnerHitBuilder innerHit) {
+    public CollapseContext(String fieldName,
+                           MappedFieldType fieldType,
+                           List<InnerHitBuilder> innerHits) {
+        this.fieldName = fieldName;
         this.fieldType = fieldType;
-        this.innerHits = Collections.singletonList(innerHit);
+        this.innerHits = innerHits;
     }
 
-    public CollapseContext(MappedFieldType fieldType, List<InnerHitBuilder> innerHits) {
-        this.fieldType = fieldType;
-        this.innerHits = innerHits;
+    /**
+     * The requested field name to collapse on.
+     */
+    public String getFieldName() {
+        return fieldName;
     }
 
     /** The field type used for collapsing **/
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java
index a1562e118fb86..3ef3064697a72 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java
@@ -61,7 +61,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
 
         if (context.collapse() != null) {
             // retrieve the `doc_value` associated with the collapse field
-            String name = context.collapse().getFieldType().name();
+            String name = context.collapse().getFieldName();
             if (context.docValueFieldsContext() == null) {
                 context.docValueFieldsContext(new DocValueFieldsContext(
                         Collections.singletonList(new FieldAndFormat(name, DocValueFieldsContext.USE_DEFAULT_FORMAT))));
diff --git a/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java b/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
index 65962e7d2c086..48ab4914e386c 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
@@ -50,7 +50,7 @@ public InternalSearchResponse(StreamInput in) throws IOException {
         super(
                 SearchHits.readSearchHits(in),
                 in.readBoolean() ? InternalAggregations.readAggregations(in) : null,
-                in.readBoolean() ? Suggest.readSuggest(in) : null,
+                in.readBoolean() ? new Suggest(in) : null,
                 in.readBoolean(),
                 in.readOptionalBoolean(),
                 in.readOptionalWriteable(SearchProfileShardResults::new),
@@ -62,7 +62,7 @@ public InternalSearchResponse(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         hits.writeTo(out);
         out.writeOptionalStreamable((InternalAggregations)aggregations);
-        out.writeOptionalStreamable(suggest);
+        out.writeOptionalWriteable(suggest);
         out.writeBoolean(timedOut);
         out.writeOptionalBoolean(terminatedEarly);
         out.writeOptionalWriteable(profileResults);
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index feb3e7876a3d7..83c43d10172c2 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -293,7 +293,7 @@ public void readFromWithId(long id, StreamInput in) throws IOException {
         pipelineAggregators = in.readNamedWriteableList(PipelineAggregator.class).stream().map(a -> (SiblingPipelineAggregator) a)
                 .collect(Collectors.toList());
         if (in.readBoolean()) {
-            suggest = Suggest.readSuggest(in);
+            suggest = new Suggest(in);
         }
         searchTimedOut = in.readBoolean();
         terminatedEarly = in.readOptionalBoolean();
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/Suggest.java b/server/src/main/java/org/elasticsearch/search/suggest/Suggest.java
index e40991933592f..c957b16502724 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/Suggest.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/Suggest.java
@@ -20,18 +20,18 @@
 
 import org.apache.lucene.util.CollectionUtil;
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
-import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -53,16 +53,15 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 /**
  * Top level suggest result, containing the result for each suggestion.
  */
-public class Suggest implements Iterable<Suggest.Suggestion<? extends Entry<? extends Option>>>, Streamable, ToXContentFragment {
+public class Suggest implements Iterable<Suggest.Suggestion<? extends Entry<? extends Option>>>, Writeable, ToXContentFragment {
 
     public static final String NAME = "suggest";
 
@@ -92,6 +91,40 @@ public Suggest(List<Suggestion<? extends Entry<? extends Option>>> suggestions)
         this.hasScoreDocs = filter(CompletionSuggestion.class).stream().anyMatch(CompletionSuggestion::hasScoreDocs);
     }
 
+    public Suggest(StreamInput in) throws IOException {
+        // in older versions, Suggestion types were serialized as Streamable
+        if (in.getVersion().before(Version.V_7_0_0_alpha1)) {
+            final int size = in.readVInt();
+            suggestions = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                Suggestion<? extends Entry<? extends Option>> suggestion;
+                final int type = in.readVInt();
+                switch (type) {
+                    case TermSuggestion.TYPE:
+                        suggestion = new TermSuggestion(in);
+                        break;
+                    case CompletionSuggestion.TYPE:
+                        suggestion = new CompletionSuggestion(in);
+                        break;
+                    case PhraseSuggestion.TYPE:
+                        suggestion = new PhraseSuggestion(in);
+                        break;
+                    default:
+                        throw new IllegalArgumentException("Unknown suggestion type with ordinal " + type);
+                }
+                suggestions.add(suggestion);
+            }
+        } else {
+            int suggestionCount = in.readVInt();
+            suggestions = new ArrayList<>(suggestionCount);
+            for (int i = 0; i < suggestionCount; i++) {
+                suggestions.add(in.readNamedWriteable(Suggestion.class));
+            }
+        }
+
+        hasScoreDocs = filter(CompletionSuggestion.class).stream().anyMatch(CompletionSuggestion::hasScoreDocs);
+    }
+
     @Override
     public Iterator<Suggestion<? extends Entry<? extends Option>>> iterator() {
         return suggestions.iterator();
@@ -125,42 +158,20 @@ public boolean hasScoreDocs() {
         return hasScoreDocs;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        final int size = in.readVInt();
-        suggestions = new ArrayList<>(size);
-        for (int i = 0; i < size; i++) {
-            // TODO: remove these complicated generics
-            Suggestion<? extends Entry<? extends Option>> suggestion;
-            final int type = in.readVInt();
-            switch (type) {
-            case TermSuggestion.TYPE:
-                suggestion = new TermSuggestion();
-                break;
-            case CompletionSuggestion.TYPE:
-                suggestion = new CompletionSuggestion();
-                break;
-            case 2: // CompletionSuggestion.TYPE
-                throw new IllegalArgumentException("Completion suggester 2.x is not supported anymore");
-            case PhraseSuggestion.TYPE:
-                suggestion = new PhraseSuggestion();
-                break;
-            default:
-                suggestion = new Suggestion();
-                break;
-            }
-            suggestion.readFrom(in);
-            suggestions.add(suggestion);
-        }
-        hasScoreDocs = filter(CompletionSuggestion.class).stream().anyMatch(CompletionSuggestion::hasScoreDocs);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(suggestions.size());
-        for (Suggestion<?> command : suggestions) {
-            out.writeVInt(command.getWriteableType());
-            command.writeTo(out);
+        // in older versions, Suggestion types were serialized as Streamable
+        if (out.getVersion().before(Version.V_7_0_0_alpha1)) {
+            out.writeVInt(suggestions.size());
+            for (Suggestion<?> command : suggestions) {
+                out.writeVInt(command.getWriteableType());
+                command.writeTo(out);
+            }
+        } else {
+            out.writeVInt(suggestions.size());
+            for (Suggestion<? extends Entry<? extends Option>> suggestion : suggestions) {
+                out.writeNamedWriteable(suggestion);
+            }
         }
     }
 
@@ -195,12 +206,6 @@ public static Suggest fromXContent(XContentParser parser) throws IOException {
         return new Suggest(suggestions);
     }
 
-    public static Suggest readSuggest(StreamInput in) throws IOException {
-        Suggest result = new Suggest();
-        result.readFrom(in);
-        return result;
-    }
-
     public static List<Suggestion<? extends Entry<? extends Option>>> reduce(Map<String, List<Suggest.Suggestion>> groupedSuggestions) {
         List<Suggestion<? extends Entry<? extends Option>>> reduced = new ArrayList<>(groupedSuggestions.size());
         for (java.util.Map.Entry<String, List<Suggestion>> unmergedResults : groupedSuggestions.entrySet()) {
@@ -232,10 +237,27 @@ public <T extends Suggestion> List<T> filter(Class<T> suggestionType) {
             .collect(Collectors.toList());
     }
 
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        return Objects.equals(suggestions, ((Suggest) other).suggestions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(suggestions);
+    }
+
     /**
      * The suggestion responses corresponding with the suggestions in the request.
      */
-    public static class Suggestion<T extends Suggestion.Entry> implements Iterable<T>, Streamable, ToXContentFragment {
+    public abstract static class Suggestion<T extends Suggestion.Entry> implements Iterable<T>, NamedWriteable, ToXContentFragment {
 
         private static final String NAME = "suggestion";
 
@@ -252,6 +274,24 @@ public Suggestion(String name, int size) {
             this.size = size; // The suggested term size specified in request, only used for merging shard responses
         }
 
+        public Suggestion(StreamInput in) throws IOException {
+            name = in.readString();
+            size = in.readVInt();
+
+            // this is a hack to work around slightly different serialization order of earlier versions of TermSuggestion
+            if (in.getVersion().before(Version.V_7_0_0_alpha1) && this instanceof TermSuggestion) {
+                TermSuggestion t = (TermSuggestion) this;
+                t.setSort(SortBy.readFromStream(in));
+            }
+
+            int entriesCount = in.readVInt();
+            entries.clear();
+            for (int i = 0; i < entriesCount; i++) {
+                T newEntry = newEntry(in);
+                entries.add(newEntry);
+            }
+        }
+
         public void addTerm(T entry) {
             entries.add(entry);
         }
@@ -259,20 +299,14 @@ public void addTerm(T entry) {
         /**
          * Returns a integer representing the type of the suggestion. This is used for
          * internal serialization over the network.
+         *
+         * This class is now serialized as a NamedWriteable and this method only remains for backwards compatibility
          */
-        public int getWriteableType() { // TODO remove this in favor of NamedWriteable
+        @Deprecated
+        public int getWriteableType() {
             return TYPE;
         }
 
-        /**
-         * Returns a string representing the type of the suggestion. This type is added to
-         * the suggestion name in the XContent response, so that it can later be used by
-         * REST clients to determine the internal type of the suggestion.
-         */
-        protected String getType() {
-            return NAME;
-        }
-
         @Override
         public Iterator<T> iterator() {
             return entries.iterator();
@@ -346,57 +380,67 @@ public void trim() {
             }
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            innerReadFrom(in);
-            int size = in.readVInt();
-            entries.clear();
-            for (int i = 0; i < size; i++) {
-                T newEntry = newEntry();
-                newEntry.readFrom(in);
-                entries.add(newEntry);
-            }
-        }
-
-        protected T newEntry() {
-            return (T)new Entry();
-        }
-
-
-        protected void innerReadFrom(StreamInput in) throws IOException {
-            name = in.readString();
-            size = in.readVInt();
-        }
+        protected abstract T newEntry();
+        protected abstract T newEntry(StreamInput in) throws IOException;
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            innerWriteTo(out);
+            out.writeString(name);
+            out.writeVInt(size);
+
+            // this is a hack to work around slightly different serialization order in older versions of TermSuggestion
+            if (out.getVersion().before(Version.V_7_0_0_alpha1) && this instanceof TermSuggestion) {
+                TermSuggestion termSuggestion = (TermSuggestion) this;
+                termSuggestion.getSort().writeTo(out);
+            }
+
             out.writeVInt(entries.size());
             for (Entry<?> entry : entries) {
                 entry.writeTo(out);
             }
         }
 
-        public void innerWriteTo(StreamOutput out) throws IOException {
-            out.writeString(name);
-            out.writeVInt(size);
-        }
+        @Override
+        public abstract String getWriteableName();
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             if (params.paramAsBoolean(RestSearchAction.TYPED_KEYS_PARAM, false)) {
                 // Concatenates the type and the name of the suggestion (ex: completion#foo)
-                builder.startArray(String.join(Aggregation.TYPED_KEYS_DELIMITER, getType(), getName()));
+                builder.startArray(String.join(Aggregation.TYPED_KEYS_DELIMITER, getWriteableName(), getName()));
             } else {
                 builder.startArray(getName());
             }
             for (Entry<?> entry : entries) {
+                builder.startObject();
                 entry.toXContent(builder, params);
+                builder.endObject();
             }
             builder.endArray();
             return builder;
         }
 
+        @Override
+        public boolean equals(Object other) {
+            if (this == other) {
+                return true;
+            }
+
+            if (other == null || getClass() != other.getClass()) {
+                return false;
+            }
+
+            Suggestion otherSuggestion = (Suggestion) other;
+            return Objects.equals(name, otherSuggestion.name)
+                && Objects.equals(size, otherSuggestion.size)
+                && Objects.equals(entries, otherSuggestion.entries);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(name, size, entries);
+        }
+
         @SuppressWarnings("unchecked")
         public static Suggestion<? extends Entry<? extends Option>> fromXContent(XContentParser parser) throws IOException {
             ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation);
@@ -417,7 +461,7 @@ protected static <E extends Suggestion.Entry<?>> void parseEntries(XContentParse
         /**
          * Represents a part from the suggest text with suggested options.
          */
-        public static class Entry<O extends Entry.Option> implements Iterable<O>, Streamable, ToXContentObject {
+        public abstract static class Entry<O extends Option> implements Iterable<O>, Writeable, ToXContentFragment {
 
             private static final String TEXT = "text";
             private static final String OFFSET = "offset";
@@ -436,7 +480,18 @@ public Entry(Text text, int offset, int length) {
                 this.length = length;
             }
 
-            protected Entry() {
+            protected Entry() {}
+
+            public Entry(StreamInput in) throws IOException {
+                text = in.readText();
+                offset = in.readVInt();
+                length = in.readVInt();
+                int suggestedWords = in.readVInt();
+                options = new ArrayList<>(suggestedWords);
+                for (int j = 0; j < suggestedWords; j++) {
+                    O newOption = newOption(in);
+                    options.add(newOption);
+                }
             }
 
             public void addOption(O option) {
@@ -534,44 +589,27 @@ void trim(int size) {
 
             @Override
             public boolean equals(Object o) {
-                if (this == o) return true;
-                if (o == null || getClass() != o.getClass()) return false;
+                if (this == o) {
+                    return true;
+                }
+                if (o == null || getClass() != o.getClass()) {
+                    return false;
+                }
 
                 Entry<?> entry = (Entry<?>) o;
-
-                if (length != entry.length) return false;
-                if (offset != entry.offset) return false;
-                if (!this.text.equals(entry.text)) return false;
-
-                return true;
+                return Objects.equals(length, entry.length)
+                    && Objects.equals(offset, entry.offset)
+                    && Objects.equals(text, entry.text)
+                    && Objects.equals(options, entry.options);
             }
 
             @Override
             public int hashCode() {
-                int result = text.hashCode();
-                result = 31 * result + offset;
-                result = 31 * result + length;
-                return result;
+                return Objects.hash(text, offset, length, options);
             }
 
-            @Override
-            public void readFrom(StreamInput in) throws IOException {
-                text = in.readText();
-                offset = in.readVInt();
-                length = in.readVInt();
-                int suggestedWords = in.readVInt();
-                options = new ArrayList<>(suggestedWords);
-                for (int j = 0; j < suggestedWords; j++) {
-                    O newOption = newOption();
-                    newOption.readFrom(in);
-                    options.add(newOption);
-                }
-            }
-
-            @SuppressWarnings("unchecked")
-            protected O newOption(){
-                return (O) new Option();
-            }
+            protected abstract O newOption();
+            protected abstract O newOption(StreamInput in) throws IOException;
 
             @Override
             public void writeTo(StreamOutput out) throws IOException {
@@ -586,40 +624,29 @@ public void writeTo(StreamOutput out) throws IOException {
 
             @Override
             public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-                builder.startObject();
                 builder.field(TEXT, text);
                 builder.field(OFFSET, offset);
                 builder.field(LENGTH, length);
                 builder.startArray(OPTIONS);
                 for (Option option : options) {
+                    builder.startObject();
                     option.toXContent(builder, params);
+                    builder.endObject();
                 }
                 builder.endArray();
-                builder.endObject();
                 return builder;
             }
 
-            private static ObjectParser<Entry<Option>, Void> PARSER = new ObjectParser<>("SuggestionEntryParser", true, Entry::new);
-
-            static {
-                declareCommonFields(PARSER);
-                PARSER.declareObjectArray(Entry::addOptions, (p,c) -> Option.fromXContent(p), new ParseField(OPTIONS));
-            }
-
             protected static void declareCommonFields(ObjectParser<? extends Entry<? extends Option>, Void> parser) {
                 parser.declareString((entry, text) -> entry.text = new Text(text), new ParseField(TEXT));
                 parser.declareInt((entry, offset) -> entry.offset = offset, new ParseField(OFFSET));
                 parser.declareInt((entry, length) -> entry.length = length, new ParseField(LENGTH));
             }
 
-            public static Entry<? extends Option> fromXContent(XContentParser parser) {
-                return PARSER.apply(parser, null);
-            }
-
             /**
              * Contains the suggested text with its document frequency and score.
              */
-            public static class Option implements Streamable, ToXContentObject {
+            public abstract static class Option implements Writeable, ToXContentFragment {
 
                 public static final ParseField TEXT = new ParseField("text");
                 public static final ParseField HIGHLIGHTED = new ParseField("highlighted");
@@ -646,7 +673,13 @@ public Option(Text text, float score) {
                     this(text, null, score);
                 }
 
-                public Option() {
+                public Option() {}
+
+                public Option(StreamInput in) throws IOException {
+                    text = in.readText();
+                    score = in.readFloat();
+                    highlighted = in.readOptionalText();
+                    collateMatch = in.readOptionalBoolean();
                 }
 
                 /**
@@ -683,14 +716,6 @@ protected void setScore(float score) {
                     this.score = score;
                 }
 
-                @Override
-                public void readFrom(StreamInput in) throws IOException {
-                    text = in.readText();
-                    score = in.readFloat();
-                    highlighted = in.readOptionalText();
-                    collateMatch = in.readOptionalBoolean();
-                }
-
                 @Override
                 public void writeTo(StreamOutput out) throws IOException {
                     out.writeText(text);
@@ -701,43 +726,17 @@ public void writeTo(StreamOutput out) throws IOException {
 
                 @Override
                 public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-                    builder.startObject();
-                    innerToXContent(builder, params);
-                    builder.endObject();
-                    return builder;
-                }
-
-                protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
                     builder.field(TEXT.getPreferredName(), text);
                     if (highlighted != null) {
                         builder.field(HIGHLIGHTED.getPreferredName(), highlighted);
                     }
+
                     builder.field(SCORE.getPreferredName(), score);
                     if (collateMatch != null) {
                         builder.field(COLLATE_MATCH.getPreferredName(), collateMatch.booleanValue());
                     }
-                    return builder;
-                }
-
-                private static final ConstructingObjectParser<Option, Void> PARSER = new ConstructingObjectParser<>("SuggestOptionParser",
-                        true, args -> {
-                            Text text = new Text((String) args[0]);
-                            float score = (Float) args[1];
-                            String highlighted = (String) args[2];
-                            Text highlightedText = highlighted == null ? null : new Text(highlighted);
-                            Boolean collateMatch = (Boolean) args[3];
-                            return new Option(text, highlightedText, score, collateMatch);
-                        });
-
-                static {
-                    PARSER.declareString(constructorArg(), TEXT);
-                    PARSER.declareFloat(constructorArg(), SCORE);
-                    PARSER.declareString(optionalConstructorArg(), HIGHLIGHTED);
-                    PARSER.declareBoolean(optionalConstructorArg(), COLLATE_MATCH);
-                }
 
-                public static Option fromXContent(XContentParser parser) {
-                    return PARSER.apply(parser, null);
+                    return builder;
                 }
 
                 protected void mergeInto(Option otherOption) {
@@ -751,18 +750,25 @@ protected void mergeInto(Option otherOption) {
                     }
                 }
 
+                /*
+                 * We consider options equal if they have the same text, even if their other fields may differ
+                 */
                 @Override
                 public boolean equals(Object o) {
-                    if (this == o) return true;
-                    if (o == null || getClass() != o.getClass()) return false;
+                    if (this == o) {
+                        return true;
+                    }
+                    if (o == null || getClass() != o.getClass()) {
+                        return false;
+                    }
 
                     Option that = (Option) o;
-                    return text.equals(that.text);
+                    return Objects.equals(text, that.text);
                 }
 
                 @Override
                 public int hashCode() {
-                    return text.hashCode();
+                    return Objects.hash(text);
                 }
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
index 6d5cd9e588a36..12ba58a2edee2 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
@@ -66,8 +66,7 @@
  */
 public final class CompletionSuggestion extends Suggest.Suggestion<CompletionSuggestion.Entry> {
 
-    public static final String NAME = "completion";
-
+    @Deprecated
     public static final int TYPE = 4;
 
     private boolean skipDuplicates;
@@ -86,14 +85,18 @@ public CompletionSuggestion(String name, int size, boolean skipDuplicates) {
         this.skipDuplicates = skipDuplicates;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
+    public CompletionSuggestion(StreamInput in) throws IOException {
+        super(in);
         if (in.getVersion().onOrAfter(Version.V_6_1_0)) {
             skipDuplicates = in.readBoolean();
         }
     }
 
+    @Override
+    public String getWriteableName() {
+        return CompletionSuggestionBuilder.SUGGESTION_NAME;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -121,6 +124,17 @@ public boolean hasScoreDocs() {
         return getOptions().size() > 0;
     }
 
+    @Override
+    public boolean equals(Object other) {
+        return super.equals(other)
+            && Objects.equals(skipDuplicates, ((CompletionSuggestion) other).skipDuplicates);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), skipDuplicates);
+    }
+
     public static CompletionSuggestion fromXContent(XContentParser parser, String name) throws IOException {
         CompletionSuggestion suggestion = new CompletionSuggestion(name, -1, false);
         parseEntries(parser, suggestion, CompletionSuggestion.Entry::fromXContent);
@@ -222,13 +236,13 @@ public int getWriteableType() {
     }
 
     @Override
-    protected String getType() {
-        return NAME;
+    protected Entry newEntry() {
+        return new Entry();
     }
 
     @Override
-    protected Entry newEntry() {
-        return new Entry();
+    protected Entry newEntry(StreamInput in) throws IOException {
+        return new Entry(in);
     }
 
     public static final class Entry extends Suggest.Suggestion.Entry<CompletionSuggestion.Entry.Option> {
@@ -237,7 +251,10 @@ public Entry(Text text, int offset, int length) {
             super(text, offset, length);
         }
 
-        Entry() {
+        Entry() {}
+
+        public Entry(StreamInput in) throws IOException {
+            super(in);
         }
 
         @Override
@@ -245,6 +262,11 @@ protected Option newOption() {
             return new Option();
         }
 
+        @Override
+        protected Option newOption(StreamInput in) throws IOException {
+            return new Option(in);
+        }
+
         private static ObjectParser<Entry, Void> PARSER = new ObjectParser<>("CompletionSuggestionEntryParser", true,
                 Entry::new);
 
@@ -274,6 +296,25 @@ protected Option() {
                 super();
             }
 
+            public Option(StreamInput in) throws IOException {
+                super(in);
+                this.doc = Lucene.readScoreDoc(in);
+                if (in.readBoolean()) {
+                    this.hit = SearchHit.readSearchHit(in);
+                }
+                int contextSize = in.readInt();
+                this.contexts = new LinkedHashMap<>(contextSize);
+                for (int i = 0; i < contextSize; i++) {
+                    String contextName = in.readString();
+                    int nContexts = in.readVInt();
+                    Set<CharSequence> contexts = new HashSet<>(nContexts);
+                    for (int j = 0; j < nContexts; j++) {
+                        contexts.add(in.readString());
+                    }
+                    this.contexts.put(contextName, contexts);
+                }
+            }
+
             @Override
             protected void mergeInto(Suggest.Suggestion.Entry.Option otherOption) {
                 // Completion suggestions are reduced by
@@ -302,7 +343,7 @@ public void setHit(SearchHit hit) {
             }
 
             @Override
-            protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
                 builder.field(TEXT.getPreferredName(), getText());
                 if (hit != null) {
                     hit.toInnerXContent(builder, params);
@@ -375,26 +416,6 @@ public static Option fromXContent(XContentParser parser) {
                 return option;
             }
 
-            @Override
-            public void readFrom(StreamInput in) throws IOException {
-                super.readFrom(in);
-                this.doc = Lucene.readScoreDoc(in);
-                if (in.readBoolean()) {
-                    this.hit = SearchHit.readSearchHit(in);
-                }
-                int contextSize = in.readInt();
-                this.contexts = new LinkedHashMap<>(contextSize);
-                for (int i = 0; i < contextSize; i++) {
-                    String contextName = in.readString();
-                    int nContexts = in.readVInt();
-                    Set<CharSequence> contexts = new HashSet<>(nContexts);
-                    for (int j = 0; j < nContexts; j++) {
-                        contexts.add(in.readString());
-                    }
-                    this.contexts.put(contextName, contexts);
-                }
-            }
-
             @Override
             public void writeTo(StreamOutput out) throws IOException {
                 super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
index 25f2f7fa382fa..6c84379c3dfdd 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
@@ -59,10 +59,12 @@
 public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSuggestionBuilder> {
 
     private static final XContentType CONTEXT_BYTES_XCONTENT_TYPE = XContentType.JSON;
-    static final String SUGGESTION_NAME = "completion";
+
     static final ParseField CONTEXTS_FIELD = new ParseField("contexts", "context");
     static final ParseField SKIP_DUPLICATES_FIELD = new ParseField("skip_duplicates");
 
+    public static final String SUGGESTION_NAME = "completion";
+
     /**
      * {
      *     "field" : STRING
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
index 670dac75ab7a0..3ff6774ff5c8b 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
@@ -133,9 +133,9 @@ public Suggestion<? extends Entry<? extends Option>> innerExecute(String name, P
                     highlighted = new Text(spare.toString());
                 }
                 if (collatePrune) {
-                    resultEntry.addOption(new Suggestion.Entry.Option(phrase, highlighted, (float) (correction.score), collateMatch));
+                    resultEntry.addOption(new PhraseSuggestion.Entry.Option(phrase, highlighted, (float) (correction.score), collateMatch));
                 } else {
-                    resultEntry.addOption(new Suggestion.Entry.Option(phrase, highlighted, (float) (correction.score)));
+                    resultEntry.addOption(new PhraseSuggestion.Entry.Option(phrase, highlighted, (float) (correction.score)));
                 }
             }
         } else {
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestion.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestion.java
index bd6c828bd42bb..39932049dfd8f 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestion.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestion.java
@@ -23,36 +23,45 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ContextParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 
 import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 /**
  * Suggestion entry returned from the {@link PhraseSuggester}.
  */
 public class PhraseSuggestion extends Suggest.Suggestion<PhraseSuggestion.Entry> {
 
-    public static final String NAME = "phrase";
+    @Deprecated
     public static final int TYPE = 3;
 
-    public PhraseSuggestion() {
-    }
+    public PhraseSuggestion() {}
 
     public PhraseSuggestion(String name, int size) {
         super(name, size);
     }
 
+    public PhraseSuggestion(StreamInput in) throws IOException {
+        super(in);
+    }
+
     @Override
-    public int getWriteableType() {
-        return TYPE;
+    public String getWriteableName() {
+        return PhraseSuggestionBuilder.SUGGESTION_NAME;
     }
 
     @Override
-    protected String getType() {
-        return NAME;
+    public int getWriteableType() {
+        return TYPE;
     }
 
     @Override
@@ -60,13 +69,18 @@ protected Entry newEntry() {
         return new Entry();
     }
 
+    @Override
+    protected Entry newEntry(StreamInput in) throws IOException {
+        return new Entry(in);
+    }
+
     public static PhraseSuggestion fromXContent(XContentParser parser, String name) throws IOException {
         PhraseSuggestion suggestion = new PhraseSuggestion(name, -1);
         parseEntries(parser, suggestion, PhraseSuggestion.Entry::fromXContent);
         return suggestion;
     }
 
-    public static class Entry extends Suggestion.Entry<Suggestion.Entry.Option> {
+    public static class Entry extends Suggestion.Entry<PhraseSuggestion.Entry.Option> {
 
         protected double cutoffScore = Double.MIN_VALUE;
 
@@ -75,7 +89,15 @@ public Entry(Text text, int offset, int length, double cutoffScore) {
             this.cutoffScore = cutoffScore;
         }
 
-        Entry() {
+        public Entry(Text text, int offset, int length) {
+            super(text, offset, length);
+        }
+
+        Entry() {}
+
+        public Entry(StreamInput in) throws IOException {
+            super(in);
+            cutoffScore = in.readDouble();
         }
 
         /**
@@ -86,7 +108,7 @@ public double getCutoffScore() {
         }
 
         @Override
-        protected void merge(Suggestion.Entry<Suggestion.Entry.Option> other) {
+        protected void merge(Suggestion.Entry<Option> other) {
             super.merge(other);
             // If the cluster contains both pre 0.90.4 and post 0.90.4 nodes then we'll see Suggestion.Entry
             // objects being merged with PhraseSuggestion.Entry objects.  We merge Suggestion.Entry objects
@@ -100,7 +122,7 @@ protected void merge(Suggestion.Entry<Suggestion.Entry.Option> other) {
         }
 
         @Override
-        public void addOption(Suggestion.Entry.Option option) {
+        public void addOption(Option option) {
             if (option.getScore() > this.cutoffScore) {
                 this.options.add(option);
             }
@@ -110,7 +132,8 @@ public void addOption(Suggestion.Entry.Option option) {
 
         static {
             declareCommonFields(PARSER);
-            PARSER.declareObjectArray(Entry::addOptions, (p,c) -> Option.fromXContent(p), new ParseField(OPTIONS));
+            PARSER.declareObjectArray(Entry::addOptions, (ContextParser<Void, Option>) (p, c) -> Option.fromXContent(p),
+                    new ParseField(OPTIONS));
         }
 
         public static Entry fromXContent(XContentParser parser) {
@@ -118,9 +141,13 @@ public static Entry fromXContent(XContentParser parser) {
         }
 
         @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            cutoffScore = in.readDouble();
+        protected Option newOption() {
+            return new Option();
+        }
+
+        @Override
+        protected Option newOption(StreamInput in) throws IOException {
+            return new Option(in);
         }
 
         @Override
@@ -128,5 +155,56 @@ public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeDouble(cutoffScore);
         }
+
+        @Override
+        public boolean equals(Object other) {
+            return super.equals(other)
+                && Objects.equals(cutoffScore, ((Entry) other).cutoffScore);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(super.hashCode(), cutoffScore);
+        }
+
+        public static class Option extends Suggestion.Entry.Option {
+
+            public Option() {
+                super();
+            }
+
+            public Option(Text text, Text highlighted, float score, Boolean collateMatch) {
+                super(text, highlighted, score, collateMatch);
+            }
+
+            public Option(Text text, Text highlighted, float score) {
+                super(text, highlighted, score);
+            }
+
+            public Option(StreamInput in) throws IOException {
+                super(in);
+            }
+
+            private static final ConstructingObjectParser<Option, Void> PARSER = new ConstructingObjectParser<>("PhraseOptionParser",
+                true, args -> {
+                    Text text = new Text((String) args[0]);
+                    float score = (Float) args[1];
+                    String highlighted = (String) args[2];
+                    Text highlightedText = highlighted == null ? null : new Text(highlighted);
+                    Boolean collateMatch = (Boolean) args[3];
+                    return new Option(text, highlightedText, score, collateMatch);
+            });
+
+            static {
+                PARSER.declareString(constructorArg(), TEXT);
+                PARSER.declareFloat(constructorArg(), SCORE);
+                PARSER.declareString(optionalConstructorArg(), HIGHLIGHTED);
+                PARSER.declareBoolean(optionalConstructorArg(), COLLATE_MATCH);
+            }
+
+            public static Option fromXContent(XContentParser parser) {
+                    return PARSER.apply(parser, null);
+            }
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
index 9b87e442ad60e..74b9437d67821 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
@@ -59,7 +59,7 @@
  */
 public class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSuggestionBuilder> {
 
-    private static final String SUGGESTION_NAME = "phrase";
+    public static final String SUGGESTION_NAME = "phrase";
 
     protected static final ParseField MAXERRORS_FIELD = new ParseField("max_errors");
     protected static final ParseField RWE_LIKELIHOOD_FIELD = new ParseField("real_word_error_likelihood");
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java b/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java
index 185e4c76b3c62..1855f1f88e5fb 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.search.suggest.term;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -28,11 +29,13 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.suggest.SortBy;
+import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
 
 import java.io.IOException;
 import java.util.Comparator;
+import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 
@@ -41,22 +44,29 @@
  */
 public class TermSuggestion extends Suggestion<TermSuggestion.Entry> {
 
-    public static final String NAME = "term";
+    @Deprecated
+    public static final int TYPE = 1;
 
     public static final Comparator<Suggestion.Entry.Option> SCORE = new Score();
     public static final Comparator<Suggestion.Entry.Option> FREQUENCY = new Frequency();
-    public static final int TYPE = 1;
 
     private SortBy sort;
 
-    public TermSuggestion() {
-    }
+    public TermSuggestion() {}
 
     public TermSuggestion(String name, int size, SortBy sort) {
         super(name, size);
         this.sort = sort;
     }
 
+    public TermSuggestion(StreamInput in) throws IOException {
+        super(in);
+
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            sort = SortBy.readFromStream(in);
+        }
+    }
+
     // Same behaviour as comparators in suggest module, but for SuggestedWord
     // Highest score first, then highest freq first, then lowest term first
     public static class Score implements Comparator<Suggestion.Entry.Option> {
@@ -103,9 +113,12 @@ public int getWriteableType() {
         return TYPE;
     }
 
-    @Override
-    protected String getType() {
-        return NAME;
+    public void setSort(SortBy sort) {
+        this.sort = sort;
+    }
+
+    public SortBy getSort() {
+        return sort;
     }
 
     @Override
@@ -121,15 +134,17 @@ protected Comparator<Option> sortComparator() {
     }
 
     @Override
-    protected void innerReadFrom(StreamInput in) throws IOException {
-        super.innerReadFrom(in);
-        sort = SortBy.readFromStream(in);
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            sort.writeTo(out);
+        }
     }
 
     @Override
-    public void innerWriteTo(StreamOutput out) throws IOException {
-        super.innerWriteTo(out);
-        sort.writeTo(out);
+    public String getWriteableName() {
+        return TermSuggestionBuilder.SUGGESTION_NAME;
     }
 
     public static TermSuggestion fromXContent(XContentParser parser, String name) throws IOException {
@@ -144,16 +159,35 @@ protected Entry newEntry() {
         return new Entry();
     }
 
+    @Override
+    protected Entry newEntry(StreamInput in) throws IOException {
+        return new Entry(in);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        return super.equals(other)
+            && Objects.equals(sort, ((TermSuggestion) other).sort);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), sort);
+    }
+
     /**
      * Represents a part from the suggest text with suggested options.
      */
-    public static class Entry extends org.elasticsearch.search.suggest.Suggest.Suggestion.Entry<TermSuggestion.Entry.Option> {
+    public static class Entry extends Suggest.Suggestion.Entry<TermSuggestion.Entry.Option> {
 
         public Entry(Text text, int offset, int length) {
             super(text, offset, length);
         }
 
-        Entry() {
+        public Entry() {}
+
+        public Entry(StreamInput in) throws IOException {
+            super(in);
         }
 
         @Override
@@ -161,6 +195,11 @@ protected Option newOption() {
             return new Option();
         }
 
+        @Override
+        protected Option newOption(StreamInput in) throws IOException {
+            return new Option(in);
+        }
+
         private static ObjectParser<Entry, Void> PARSER = new ObjectParser<>("TermSuggestionEntryParser", true, Entry::new);
 
         static {
@@ -175,7 +214,7 @@ public static Entry fromXContent(XContentParser parser) {
         /**
          * Contains the suggested text with its document frequency and score.
          */
-        public static class Option extends org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option {
+        public static class Option extends Suggest.Suggestion.Entry.Option {
 
             public static final ParseField FREQ = new ParseField("freq");
 
@@ -186,6 +225,11 @@ public Option(Text text, int freq, float score) {
                 this.freq = freq;
             }
 
+            public Option(StreamInput in) throws IOException {
+                super(in);
+                freq = in.readVInt();
+            }
+
             @Override
             protected void mergeInto(Suggestion.Entry.Option otherOption) {
                 super.mergeInto(otherOption);
@@ -207,12 +251,6 @@ public int getFreq() {
                 return freq;
             }
 
-            @Override
-            public void readFrom(StreamInput in) throws IOException {
-                super.readFrom(in);
-                freq = in.readVInt();
-            }
-
             @Override
             public void writeTo(StreamOutput out) throws IOException {
                 super.writeTo(out);
@@ -220,8 +258,8 @@ public void writeTo(StreamOutput out) throws IOException {
             }
 
             @Override
-            protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-                builder = super.innerToXContent(builder, params);
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                builder = super.toXContent(builder, params);
                 builder.field(FREQ.getPreferredName(), freq);
                 return builder;
             }
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java b/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
index 5d880c55ff047..fdc31dd6c2fca 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
@@ -67,7 +67,7 @@
  */
 public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuilder> {
 
-    private static final String SUGGESTION_NAME = "term";
+    public static final String SUGGESTION_NAME = "term";
 
     private SuggestMode suggestMode = SuggestMode.MISSING;
     private float accuracy = DEFAULT_ACCURACY;
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java
index 1578cb0a289ee..f2bdc2ba5df4f 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java
@@ -40,15 +40,11 @@
 /**
  * Stores information about failures that occurred during shard snapshotting process
  */
-public class SnapshotShardFailure implements ShardOperationFailedException {
-    private ShardId shardId;
-
-    private String reason;
+public class SnapshotShardFailure extends ShardOperationFailedException {
 
     @Nullable
     private String nodeId;
-
-    private RestStatus status;
+    private ShardId shardId;
 
     private SnapshotShardFailure() {
 
@@ -74,56 +70,9 @@ public SnapshotShardFailure(@Nullable String nodeId, ShardId shardId, String rea
      * @param status  rest status
      */
     private SnapshotShardFailure(@Nullable String nodeId, ShardId shardId, String reason, RestStatus status) {
-        assert reason != null;
+        super(shardId.getIndexName(), shardId.id(), reason, status, new IndexShardSnapshotFailedException(shardId, reason));
         this.nodeId = nodeId;
         this.shardId = shardId;
-        this.reason = reason;
-        this.status = status;
-    }
-
-    /**
-     * Returns index where failure occurred
-     *
-     * @return index
-     */
-    @Override
-    public String index() {
-        return this.shardId.getIndexName();
-    }
-
-    /**
-     * Returns shard id where failure occurred
-     *
-     * @return shard id
-     */
-    @Override
-    public int shardId() {
-        return this.shardId.id();
-    }
-
-    /**
-     * Returns reason for the failure
-     *
-     * @return reason for the failure
-     */
-    @Override
-    public String reason() {
-        return this.reason;
-    }
-
-    /**
-     * Returns {@link RestStatus} corresponding to this failure
-     *
-     * @return REST status
-     */
-    @Override
-    public RestStatus status() {
-        return status;
-    }
-
-    @Override
-    public Throwable getCause() {
-        return new IndexShardSnapshotFailedException(shardId, reason);
     }
 
     /**
@@ -142,7 +91,7 @@ public String nodeId() {
      * @param in stream input
      * @return shard failure information
      */
-    public static SnapshotShardFailure readSnapshotShardFailure(StreamInput in) throws IOException {
+    static SnapshotShardFailure readSnapshotShardFailure(StreamInput in) throws IOException {
         SnapshotShardFailure exp = new SnapshotShardFailure();
         exp.readFrom(in);
         return exp;
@@ -152,6 +101,8 @@ public static SnapshotShardFailure readSnapshotShardFailure(StreamInput in) thro
     public void readFrom(StreamInput in) throws IOException {
         nodeId = in.readOptionalString();
         shardId = ShardId.readShardId(in);
+        super.shardId = shardId.getId();
+        super.index = shardId.getIndexName();
         reason = in.readString();
         status = RestStatus.readFrom(in);
     }
diff --git a/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java b/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java
new file mode 100644
index 0000000000000..cdc68dc10f42e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.concurrent.CompletableContext;
+
+
+/**
+ * Abstract Transport.Connection that provides common close logic.
+ */
+public abstract class CloseableConnection implements Transport.Connection {
+
+    private final CompletableContext<Void> closeContext = new CompletableContext<>();
+
+    @Override
+    public void addCloseListener(ActionListener<Void> listener) {
+        closeContext.addListener(ActionListener.toBiConsumer(listener));
+    }
+
+    @Override
+    public boolean isClosed() {
+        return closeContext.isDone();
+    }
+
+    @Override
+    public void close() {
+        // This method is safe to call multiple times as the close context will provide concurrency
+        // protection and only be completed once. The attached listeners will only be notified once.
+        closeContext.complete(null);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java
new file mode 100644
index 0000000000000..f2b89ff59775c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java
@@ -0,0 +1,282 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.component.Lifecycle;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractLifecycleRunnable;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.util.concurrent.KeyedLock;
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This class manages node connections. The connection is opened by the underlying transport. Once the
+ * connection is opened, this class manages the connection. This includes keep-alive pings and closing
+ * the connection when the connection manager is closed.
+ */
+public class ConnectionManager implements Closeable {
+
+    private final ConcurrentMap<DiscoveryNode, Transport.Connection> connectedNodes = ConcurrentCollections.newConcurrentMap();
+    private final KeyedLock<String> connectionLock = new KeyedLock<>();
+    private final Logger logger;
+    private final Transport transport;
+    private final ThreadPool threadPool;
+    private final TimeValue pingSchedule;
+    private final Lifecycle lifecycle = new Lifecycle();
+    private final ReadWriteLock closeLock = new ReentrantReadWriteLock();
+    private final DelegatingNodeConnectionListener connectionListener = new DelegatingNodeConnectionListener();
+
+    public ConnectionManager(Settings settings, Transport transport, ThreadPool threadPool) {
+        this.logger = Loggers.getLogger(getClass(), settings);
+        this.transport = transport;
+        this.threadPool = threadPool;
+        this.pingSchedule = TcpTransport.PING_SCHEDULE.get(settings);
+        this.lifecycle.moveToStarted();
+
+        if (pingSchedule.millis() > 0) {
+            threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, new ScheduledPing());
+        }
+    }
+
+    public void addListener(TransportConnectionListener listener) {
+        this.connectionListener.listeners.add(listener);
+    }
+
+    public void removeListener(TransportConnectionListener listener) {
+        this.connectionListener.listeners.remove(listener);
+    }
+
+    /**
+     * Connects to a node with the given connection profile. If the node is already connected this method has no effect.
+     * Once a successful is established, it can be validated before being exposed.
+     */
+    public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                              CheckedBiConsumer<Transport.Connection, ConnectionProfile, IOException> connectionValidator)
+        throws ConnectTransportException {
+        if (node == null) {
+            throw new ConnectTransportException(null, "can't connect to a null node");
+        }
+        closeLock.readLock().lock(); // ensure we don't open connections while we are closing
+        try {
+            ensureOpen();
+            try (Releasable ignored = connectionLock.acquire(node.getId())) {
+                Transport.Connection connection = connectedNodes.get(node);
+                if (connection != null) {
+                    return;
+                }
+                boolean success = false;
+                try {
+                    connection = transport.openConnection(node, connectionProfile);
+                    connectionValidator.accept(connection, connectionProfile);
+                    // we acquire a connection lock, so no way there is an existing connection
+                    connectedNodes.put(node, connection);
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("connected to node [{}]", node);
+                    }
+                    try {
+                        connectionListener.onNodeConnected(node);
+                    } finally {
+                        final Transport.Connection finalConnection = connection;
+                        connection.addCloseListener(ActionListener.wrap(() -> {
+                            connectedNodes.remove(node, finalConnection);
+                            connectionListener.onNodeDisconnected(node);
+                        }));
+                    }
+                    if (connection.isClosed()) {
+                        throw new NodeNotConnectedException(node, "connection concurrently closed");
+                    }
+                    success = true;
+                } catch (ConnectTransportException e) {
+                    throw e;
+                } catch (Exception e) {
+                    throw new ConnectTransportException(node, "general node connection failure", e);
+                } finally {
+                    if (success == false) { // close the connection if there is a failure
+                        logger.trace(() -> new ParameterizedMessage("failed to connect to [{}], cleaning dangling connections", node));
+                        IOUtils.closeWhileHandlingException(connection);
+                    }
+                }
+            }
+        } finally {
+            closeLock.readLock().unlock();
+        }
+    }
+
+    /**
+     * Returns a connection for the given node if the node is connected.
+     * Connections returned from this method must not be closed. The lifecycle of this connection is
+     * maintained by this connection manager
+     *
+     * @throws NodeNotConnectedException if the node is not connected
+     * @see #connectToNode(DiscoveryNode, ConnectionProfile, CheckedBiConsumer)
+     */
+    public Transport.Connection getConnection(DiscoveryNode node) {
+        Transport.Connection connection = connectedNodes.get(node);
+        if (connection == null) {
+            throw new NodeNotConnectedException(node, "Node not connected");
+        }
+        return connection;
+    }
+
+    /**
+     * Returns {@code true} if the node is connected.
+     */
+    public boolean nodeConnected(DiscoveryNode node) {
+        return connectedNodes.containsKey(node);
+    }
+
+    /**
+     * Disconnected from the given node, if not connected, will do nothing.
+     */
+    public void disconnectFromNode(DiscoveryNode node) {
+        Transport.Connection nodeChannels = connectedNodes.remove(node);
+        if (nodeChannels != null) {
+            // if we found it and removed it we close
+            nodeChannels.close();
+        }
+    }
+
+    public int connectedNodeCount() {
+        return connectedNodes.size();
+    }
+
+    @Override
+    public void close() {
+        lifecycle.moveToStopped();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        // TODO: Consider moving all read/write lock (in Transport and this class) to the TransportService
+        threadPool.generic().execute(() -> {
+            closeLock.writeLock().lock();
+            try {
+                // we are holding a write lock so nobody modifies the connectedNodes / openConnections map - it's safe to first close
+                // all instances and then clear them maps
+                Iterator<Map.Entry<DiscoveryNode, Transport.Connection>> iterator = connectedNodes.entrySet().iterator();
+                while (iterator.hasNext()) {
+                    Map.Entry<DiscoveryNode, Transport.Connection> next = iterator.next();
+                    try {
+                        IOUtils.closeWhileHandlingException(next.getValue());
+                    } finally {
+                        iterator.remove();
+                    }
+                }
+            } finally {
+                closeLock.writeLock().unlock();
+                latch.countDown();
+            }
+        });
+
+        try {
+            try {
+                latch.await(30, TimeUnit.SECONDS);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                // ignore
+            }
+        } finally {
+            lifecycle.moveToClosed();
+        }
+    }
+
+    private void ensureOpen() {
+        if (lifecycle.started() == false) {
+            throw new IllegalStateException("connection manager is closed");
+        }
+    }
+
+    private class ScheduledPing extends AbstractLifecycleRunnable {
+
+        private ScheduledPing() {
+            super(lifecycle, logger);
+        }
+
+        @Override
+        protected void doRunInLifecycle() {
+            for (Map.Entry<DiscoveryNode, Transport.Connection> entry : connectedNodes.entrySet()) {
+                Transport.Connection connection = entry.getValue();
+                if (connection.sendPing() == false) {
+                    logger.warn("attempted to send ping to connection without support for pings [{}]", connection);
+                }
+            }
+        }
+
+        @Override
+        protected void onAfterInLifecycle() {
+            try {
+                threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, this);
+            } catch (EsRejectedExecutionException ex) {
+                if (ex.isExecutorShutdown()) {
+                    logger.debug("couldn't schedule new ping execution, executor is shutting down", ex);
+                } else {
+                    throw ex;
+                }
+            }
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            if (lifecycle.stoppedOrClosed()) {
+                logger.trace("failed to send ping transport message", e);
+            } else {
+                logger.warn("failed to send ping transport message", e);
+            }
+        }
+    }
+
+    private static final class DelegatingNodeConnectionListener implements TransportConnectionListener {
+
+        private final List<TransportConnectionListener> listeners = new CopyOnWriteArrayList<>();
+
+        @Override
+        public void onNodeDisconnected(DiscoveryNode key) {
+            for (TransportConnectionListener listener : listeners) {
+                listener.onNodeDisconnected(key);
+            }
+        }
+
+        @Override
+        public void onNodeConnected(DiscoveryNode node) {
+            for (TransportConnectionListener listener : listeners) {
+                listener.onNodeConnected(node);
+            }
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
index 91d7b1b372e51..355a9c655c998 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
@@ -289,11 +289,26 @@ public void sendRequest(long requestId, String action, TransportRequest request,
                     TransportActionProxy.wrapRequest(targetNode, request), options);
         }
 
+        @Override
+        public boolean sendPing() {
+            return proxyConnection.sendPing();
+        }
+
         @Override
         public void close() {
             assert false: "proxy connections must not be closed";
         }
 
+        @Override
+        public void addCloseListener(ActionListener<Void> listener) {
+            proxyConnection.addCloseListener(listener);
+        }
+
+        @Override
+        public boolean isClosed() {
+            return proxyConnection.isClosed();
+        }
+
         @Override
         public Version getVersion() {
             return proxyConnection.getVersion();
diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
index b2294ce589325..064c76cc03fce 100644
--- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Booleans;
-import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.breaker.CircuitBreaker;
@@ -48,7 +47,6 @@
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.network.CloseableChannel;
@@ -63,10 +61,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.concurrent.AbstractLifecycleRunnable;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.common.util.concurrent.KeyedLock;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -90,7 +85,6 @@
 import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -190,6 +184,8 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
     // This is the number of bytes necessary to read the message size
     public static final int BYTES_NEEDED_FOR_MESSAGE_SIZE = TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE;
     public static final int PING_DATA_SIZE = -1;
+    protected final CounterMetric successfulPings = new CounterMetric();
+    protected final CounterMetric failedPings = new CounterMetric();
     private static final long NINETY_PER_HEAP_SIZE = (long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.9);
     private static final BytesReference EMPTY_BYTES_REFERENCE = new BytesArray(new byte[0]);
 
@@ -198,9 +194,6 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
     private final String[] features;
 
     private final CircuitBreakerService circuitBreakerService;
-    // package visibility for tests
-    protected final ScheduledPing scheduledPing;
-    private final TimeValue pingSchedule;
     protected final ThreadPool threadPool;
     private final BigArrays bigArrays;
     protected final NetworkService networkService;
@@ -209,16 +202,13 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
     private final DelegatingTransportConnectionListener transportListener = new DelegatingTransportConnectionListener();
 
     private final ConcurrentMap<String, BoundTransportAddress> profileBoundAddresses = newConcurrentMap();
-    // node id to actual channel
-    private final ConcurrentMap<DiscoveryNode, NodeChannels> connectedNodes = newConcurrentMap();
     private final Map<String, List<TcpServerChannel>> serverChannels = newConcurrentMap();
     private final Set<TcpChannel> acceptedChannels = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
-    private final KeyedLock<String> connectionLock = new KeyedLock<>();
     private final NamedWriteableRegistry namedWriteableRegistry;
 
     // this lock is here to make sure we close this transport and disconnect all the client nodes
-    // connections while no connect operations is going on... (this might help with 100% CPU when stopping the transport?)
+    // connections while no connect operations is going on
     private final ReadWriteLock closeLock = new ReentrantReadWriteLock();
     protected final boolean compress;
     private volatile BoundTransportAddress boundAddress;
@@ -233,6 +223,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
     private final MeanMetric transmittedBytesMetric = new MeanMetric();
     private volatile Map<String, RequestHandlerRegistry> requestHandlers = Collections.emptyMap();
     private final ResponseHandlers responseHandlers = new ResponseHandlers();
+    private final BytesReference pingMessage;
 
     public TcpTransport(String transportName, Settings settings, ThreadPool threadPool, BigArrays bigArrays,
                         CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry,
@@ -242,8 +233,6 @@ public TcpTransport(String transportName, Settings settings, ThreadPool threadPo
         this.threadPool = threadPool;
         this.bigArrays = bigArrays;
         this.circuitBreakerService = circuitBreakerService;
-        this.scheduledPing = new ScheduledPing();
-        this.pingSchedule = PING_SCHEDULE.get(settings);
         this.namedWriteableRegistry = namedWriteableRegistry;
         this.compress = Transport.TRANSPORT_TCP_COMPRESS.get(settings);
         this.networkService = networkService;
@@ -261,6 +250,15 @@ public TcpTransport(String transportName, Settings settings, ThreadPool threadPo
             // use a sorted set to present the features in a consistent order
             this.features = new TreeSet<>(defaultFeatures.names()).toArray(new String[defaultFeatures.names().size()]);
         }
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.writeByte((byte) 'E');
+            out.writeByte((byte) 'S');
+            out.writeInt(TcpTransport.PING_DATA_SIZE);
+            pingMessage = out.bytes();
+        } catch (IOException e) {
+            throw new AssertionError(e.getMessage(), e); // won't happen
+        }
     }
 
     static ConnectionProfile buildDefaultConnectionProfile(Settings settings) {
@@ -284,9 +282,6 @@ static ConnectionProfile buildDefaultConnectionProfile(Settings settings) {
 
     @Override
     protected void doStart() {
-        if (pingSchedule.millis() > 0) {
-            threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, scheduledPing);
-        }
     }
 
     @Override
@@ -348,92 +343,12 @@ public String executor() {
         }
     }
 
-    public class ScheduledPing extends AbstractLifecycleRunnable {
-
-        /**
-         * The magic number (must be lower than 0) for a ping message.
-         */
-        private final BytesReference pingHeader;
-        final CounterMetric successfulPings = new CounterMetric();
-        final CounterMetric failedPings = new CounterMetric();
-
-        public ScheduledPing() {
-            super(lifecycle, logger);
-            try (BytesStreamOutput out = new BytesStreamOutput()) {
-                out.writeByte((byte) 'E');
-                out.writeByte((byte) 'S');
-                out.writeInt(PING_DATA_SIZE);
-                pingHeader = out.bytes();
-            } catch (IOException e) {
-                throw new IllegalStateException(e.getMessage(), e); // won't happen
-            }
-        }
-
-        @Override
-        protected void doRunInLifecycle() throws Exception {
-            for (Map.Entry<DiscoveryNode, NodeChannels> entry : connectedNodes.entrySet()) {
-                DiscoveryNode node = entry.getKey();
-                NodeChannels channels = entry.getValue();
-                for (TcpChannel channel : channels.getChannels()) {
-                    internalSendMessage(channel, pingHeader, new SendMetricListener(pingHeader.length()) {
-                        @Override
-                        protected void innerInnerOnResponse(Void v) {
-                            successfulPings.inc();
-                        }
-
-                        @Override
-                        protected void innerOnFailure(Exception e) {
-                            if (channel.isOpen()) {
-                                logger.debug(() -> new ParameterizedMessage("[{}] failed to send ping transport message", node), e);
-                                failedPings.inc();
-                            } else {
-                                logger.trace(() ->
-                                    new ParameterizedMessage("[{}] failed to send ping transport message (channel closed)", node), e);
-                            }
-
-                        }
-                    });
-                }
-            }
-        }
-
-        public long getSuccessfulPings() {
-            return successfulPings.count();
-        }
-
-        public long getFailedPings() {
-            return failedPings.count();
-        }
-
-        @Override
-        protected void onAfterInLifecycle() {
-            try {
-                threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, this);
-            } catch (EsRejectedExecutionException ex) {
-                if (ex.isExecutorShutdown()) {
-                    logger.debug("couldn't schedule new ping execution, executor is shutting down", ex);
-                } else {
-                    throw ex;
-                }
-            }
-        }
-
-        @Override
-        public void onFailure(Exception e) {
-            if (lifecycle.stoppedOrClosed()) {
-                logger.trace("failed to send ping transport message", e);
-            } else {
-                logger.warn("failed to send ping transport message", e);
-            }
-        }
-    }
-
-    public final class NodeChannels implements Connection {
+    public final class NodeChannels extends CloseableConnection {
         private final Map<TransportRequestOptions.Type, ConnectionProfile.ConnectionTypeHandle> typeMapping;
         private final List<TcpChannel> channels;
         private final DiscoveryNode node;
-        private final AtomicBoolean closed = new AtomicBoolean(false);
         private final Version version;
+        private final AtomicBoolean isClosing = new AtomicBoolean(false);
 
         NodeChannels(DiscoveryNode node, List<TcpChannel> channels, ConnectionProfile connectionProfile, Version handshakeVersion) {
             this.node = node;
@@ -465,13 +380,38 @@ public TcpChannel channel(TransportRequestOptions.Type type) {
             return connectionTypeHandle.getChannel(channels);
         }
 
-        public boolean allChannelsOpen() {
+        boolean allChannelsOpen() {
             return channels.stream().allMatch(TcpChannel::isOpen);
         }
 
+        @Override
+        public boolean sendPing() {
+            for (TcpChannel channel : channels) {
+                internalSendMessage(channel, pingMessage, new SendMetricListener(pingMessage.length()) {
+                    @Override
+                    protected void innerInnerOnResponse(Void v) {
+                        successfulPings.inc();
+                    }
+
+                    @Override
+                    protected void innerOnFailure(Exception e) {
+                        if (channel.isOpen()) {
+                            logger.debug(() -> new ParameterizedMessage("[{}] failed to send ping transport message", node), e);
+                            failedPings.inc();
+                        } else {
+                            logger.trace(() ->
+                                new ParameterizedMessage("[{}] failed to send ping transport message (channel closed)", node), e);
+                        }
+
+                    }
+                });
+            }
+            return true;
+        }
+
         @Override
         public void close() {
-            if (closed.compareAndSet(false, true)) {
+            if (isClosing.compareAndSet(false, true)) {
                 try {
                     if (lifecycle.stopped()) {
                         /* We set SO_LINGER timeout to 0 to ensure that when we shutdown the node we don't
@@ -494,7 +434,8 @@ public void close() {
                     boolean block = lifecycle.stopped() && Transports.isTransportThread(Thread.currentThread()) == false;
                     CloseableChannel.closeChannels(channels, block);
                 } finally {
-                    transportListener.onConnectionClosed(this);
+                    // Call the super method to trigger listeners
+                    super.close();
                 }
             }
         }
@@ -507,88 +448,19 @@ public DiscoveryNode getNode() {
         @Override
         public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
             throws IOException, TransportException {
-            if (closed.get()) {
+            if (isClosing.get()) {
                 throw new NodeNotConnectedException(node, "connection already closed");
             }
             TcpChannel channel = channel(options.type());
             sendRequestToChannel(this.node, channel, requestId, action, request, options, getVersion(), (byte) 0);
         }
-
-        boolean isClosed() {
-            return closed.get();
-        }
-    }
-
-    @Override
-    public boolean nodeConnected(DiscoveryNode node) {
-        return connectedNodes.containsKey(node);
-    }
-
-    @Override
-    public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
-                              CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
-        throws ConnectTransportException {
-        connectionProfile = resolveConnectionProfile(connectionProfile);
-        if (node == null) {
-            throw new ConnectTransportException(null, "can't connect to a null node");
-        }
-        closeLock.readLock().lock(); // ensure we don't open connections while we are closing
-        try {
-            ensureOpen();
-            try (Releasable ignored = connectionLock.acquire(node.getId())) {
-                NodeChannels nodeChannels = connectedNodes.get(node);
-                if (nodeChannels != null) {
-                    return;
-                }
-                boolean success = false;
-                try {
-                    nodeChannels = openConnection(node, connectionProfile);
-                    connectionValidator.accept(nodeChannels, connectionProfile);
-                    // we acquire a connection lock, so no way there is an existing connection
-                    connectedNodes.put(node, nodeChannels);
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("connected to node [{}]", node);
-                    }
-                    try {
-                        transportListener.onNodeConnected(node);
-                    } finally {
-                        if (nodeChannels.isClosed()) {
-                            // we got closed concurrently due to a disconnect or some other event on the channel.
-                            // the close callback will close the NodeChannel instance first and then try to remove
-                            // the connection from the connected nodes. It will NOT acquire the connectionLock for
-                            // the node to prevent any blocking calls on network threads. Yet, we still establish a happens
-                            // before relationship to the connectedNodes.put since we check if we can remove the
-                            // (DiscoveryNode, NodeChannels) tuple from the map after we closed. Here we check if it's closed an if so we
-                            // try to remove it first either way one of the two wins even if the callback has run before we even added the
-                            // tuple to the map since in that case we remove it here again
-                            if (connectedNodes.remove(node, nodeChannels)) {
-                                transportListener.onNodeDisconnected(node);
-                            }
-                            throw new NodeNotConnectedException(node, "connection concurrently closed");
-                        }
-                    }
-                    success = true;
-                } catch (ConnectTransportException e) {
-                    throw e;
-                } catch (Exception e) {
-                    throw new ConnectTransportException(node, "general node connection failure", e);
-                } finally {
-                    if (success == false) { // close the connection if there is a failure
-                        logger.trace(() -> new ParameterizedMessage("failed to connect to [{}], cleaning dangling connections", node));
-                        IOUtils.closeWhileHandlingException(nodeChannels);
-                    }
-                }
-            }
-        } finally {
-            closeLock.readLock().unlock();
-        }
     }
 
     /**
      * takes a {@link ConnectionProfile} that have been passed as a parameter to the public methods
      * and resolves it to a fully specified (i.e., no nulls) profile
      */
-    static ConnectionProfile resolveConnectionProfile(@Nullable ConnectionProfile connectionProfile,
+    protected static ConnectionProfile resolveConnectionProfile(@Nullable ConnectionProfile connectionProfile,
                                                       ConnectionProfile defaultConnectionProfile) {
         Objects.requireNonNull(defaultConnectionProfile);
         if (connectionProfile == null) {
@@ -612,7 +484,7 @@ protected ConnectionProfile resolveConnectionProfile(ConnectionProfile connectio
     }
 
     @Override
-    public final NodeChannels openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) {
+    public NodeChannels openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) {
         if (node == null) {
             throw new ConnectTransportException(null, "can't open connection to a null node");
         }
@@ -664,16 +536,16 @@ public final NodeChannels openConnection(DiscoveryNode node, ConnectionProfile c
                 // At this point we should construct the connection, notify the transport service, and attach close listeners to the
                 // underlying channels.
                 nodeChannels = new NodeChannels(node, channels, connectionProfile, version);
-                transportListener.onConnectionOpened(nodeChannels);
                 final NodeChannels finalNodeChannels = nodeChannels;
-                final AtomicBoolean runOnce = new AtomicBoolean(false);
+                try {
+                    transportListener.onConnectionOpened(nodeChannels);
+                } finally {
+                    nodeChannels.addCloseListener(ActionListener.wrap(() -> transportListener.onConnectionClosed(finalNodeChannels)));
+                }
+
                 Consumer<TcpChannel> onClose = c -> {
                     assert c.isOpen() == false : "channel is still open when onClose is called";
-                    // we only need to disconnect from the nodes once since all other channels
-                    // will also try to run this we protect it from running multiple times.
-                    if (runOnce.compareAndSet(false, true)) {
-                        disconnectFromNodeCloseAndNotify(node, finalNodeChannels);
-                    }
+                    finalNodeChannels.close();
                 };
 
                 nodeChannels.channels.forEach(ch -> ch.addCloseListener(ActionListener.wrap(() -> onClose.accept(ch))));
@@ -699,46 +571,6 @@ public final NodeChannels openConnection(DiscoveryNode node, ConnectionProfile c
         }
     }
 
-    private void disconnectFromNodeCloseAndNotify(DiscoveryNode node, NodeChannels nodeChannels) {
-        assert nodeChannels != null : "nodeChannels must not be null";
-        try {
-            IOUtils.closeWhileHandlingException(nodeChannels);
-        } finally {
-            if (closeLock.readLock().tryLock()) {
-                try {
-                    if (connectedNodes.remove(node, nodeChannels)) {
-                        transportListener.onNodeDisconnected(node);
-                    }
-                } finally {
-                    closeLock.readLock().unlock();
-                }
-            }
-        }
-    }
-
-    @Override
-    public NodeChannels getConnection(DiscoveryNode node) {
-        NodeChannels nodeChannels = connectedNodes.get(node);
-        if (nodeChannels == null) {
-            throw new NodeNotConnectedException(node, "Node not connected");
-        }
-        return nodeChannels;
-    }
-
-    @Override
-    public void disconnectFromNode(DiscoveryNode node) {
-        closeLock.readLock().lock();
-        NodeChannels nodeChannels = null;
-        try (Releasable ignored = connectionLock.acquire(node.getId())) {
-            nodeChannels = connectedNodes.remove(node);
-        } finally {
-            closeLock.readLock().unlock();
-            if (nodeChannels != null) { // if we found it and removed it we close and notify
-                IOUtils.closeWhileHandlingException(nodeChannels, () -> transportListener.onNodeDisconnected(node));
-            }
-        }
-    }
-
     protected Version getCurrentVersion() {
         // this is just for tests to mock stuff like the nodes version - tests can override this internally
         return Version.CURRENT;
@@ -983,19 +815,6 @@ protected final void doStop() {
                 CloseableChannel.closeChannels(new ArrayList<>(acceptedChannels), true);
                 acceptedChannels.clear();
 
-
-                // we are holding a write lock so nobody modifies the connectedNodes / openConnections map - it's safe to first close
-                // all instances and then clear them maps
-                Iterator<Map.Entry<DiscoveryNode, NodeChannels>> iterator = connectedNodes.entrySet().iterator();
-                while (iterator.hasNext()) {
-                    Map.Entry<DiscoveryNode, NodeChannels> next = iterator.next();
-                    try {
-                        IOUtils.closeWhileHandlingException(next.getValue());
-                        transportListener.onNodeDisconnected(next.getKey());
-                    } finally {
-                        iterator.remove();
-                    }
-                }
                 stopInternal();
             } finally {
                 closeLock.writeLock().unlock();
@@ -1111,8 +930,7 @@ public boolean canCompress(TransportRequest request) {
 
     private void sendRequestToChannel(final DiscoveryNode node, final TcpChannel channel, final long requestId, final String action,
                                       final TransportRequest request, TransportRequestOptions options, Version channelVersion,
-                                      byte status) throws IOException,
-        TransportException {
+                                      byte status) throws IOException, TransportException {
         if (compress) {
             options = TransportRequestOptions.builder(options).withCompress(true).build();
         }
diff --git a/server/src/main/java/org/elasticsearch/transport/Transport.java b/server/src/main/java/org/elasticsearch/transport/Transport.java
index 74235479657bf..9538119f43b8a 100644
--- a/server/src/main/java/org/elasticsearch/transport/Transport.java
+++ b/server/src/main/java/org/elasticsearch/transport/Transport.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.component.LifecycleComponent;
@@ -85,23 +85,6 @@ public interface Transport extends LifecycleComponent {
      */
     TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException;
 
-    /**
-     * Returns {@code true} if the node is connected.
-     */
-    boolean nodeConnected(DiscoveryNode node);
-
-    /**
-     * Connects to a node with the given connection profile. If the node is already connected this method has no effect.
-     * Once a successful is established, it can be validated before being exposed.
-     */
-    void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
-                       CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator) throws ConnectTransportException;
-
-    /**
-     * Disconnected from the given node, if not connected, will do nothing.
-     */
-    void disconnectFromNode(DiscoveryNode node);
-
     /**
      * Returns a list of all local adresses for this transport
      */
@@ -112,23 +95,10 @@ default CircuitBreaker getInFlightRequestBreaker() {
     }
 
     /**
-     * Returns a connection for the given node if the node is connected.
-     * Connections returned from this method must not be closed. The lifecycle of this connection is maintained by the Transport
-     * implementation.
-     *
-     * @throws NodeNotConnectedException if the node is not connected
-     * @see #connectToNode(DiscoveryNode, ConnectionProfile, CheckedBiConsumer)
-     */
-    Connection getConnection(DiscoveryNode node);
-
-    /**
-     * Opens a new connection to the given node and returns it. In contrast to
-     * {@link #connectToNode(DiscoveryNode, ConnectionProfile, CheckedBiConsumer)} the returned connection is not managed by
+     * Opens a new connection to the given node and returns it. The returned connection is not managed by
      * the transport implementation. This connection must be closed once it's not needed anymore.
-     * This connection type can be used to execute a handshake between two nodes before the node will be published via
-     * {@link #connectToNode(DiscoveryNode, ConnectionProfile, CheckedBiConsumer)}.
      */
-    Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException;
+    Connection openConnection(DiscoveryNode node, ConnectionProfile profile);
 
     TransportStats getStats();
 
@@ -154,6 +124,21 @@ interface Connection extends Closeable {
         void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) throws
             IOException, TransportException;
 
+        default boolean sendPing() {
+            return false;
+        }
+
+        /**
+         * The listener's {@link ActionListener#onResponse(Object)} method will be called when this
+         * connection is closed. No implementations currently throw an exception during close, so
+         * {@link ActionListener#onFailure(Exception)} will not be called.
+         *
+         * @param listener to be called
+         */
+        void addCloseListener(ActionListener<Void> listener);
+
+        boolean isClosed();
+
         /**
          * Returns the version of the node this connection was established with.
          */
@@ -168,6 +153,9 @@ default Version getVersion() {
         default Object getCacheKey() {
             return this;
         }
+
+        @Override
+        void close();
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java
index 032258101f730..d7ece36d7fdda 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -21,11 +21,11 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
@@ -47,6 +47,7 @@
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.tasks.TaskManager;
@@ -79,6 +80,7 @@ public class TransportService extends AbstractLifecycleComponent implements Tran
 
     private final CountDownLatch blockIncomingRequestsLatch = new CountDownLatch(1);
     protected final Transport transport;
+    protected final ConnectionManager connectionManager;
     protected final ThreadPool threadPool;
     protected final ClusterName clusterName;
     protected final TaskManager taskManager;
@@ -109,6 +111,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) {
             Function.identity(), Property.Dynamic, Property.NodeScope);
 
     private final Logger tracerLog;
+    private final ConnectionProfile defaultConnectionProfile;
 
     volatile String[] tracerLogInclude;
     volatile String[] tracerLogExclude;
@@ -131,6 +134,15 @@ public void sendRequest(long requestId, String action, TransportRequest request,
             sendLocalRequest(requestId, action, request, options);
         }
 
+        @Override
+        public void addCloseListener(ActionListener<Void> listener) {
+        }
+
+        @Override
+        public boolean isClosed() {
+            return false;
+        }
+
         @Override
         public void close() {
         }
@@ -145,6 +157,13 @@ public void close() {
     public TransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor transportInterceptor,
                             Function<BoundTransportAddress, DiscoveryNode> localNodeFactory, @Nullable ClusterSettings clusterSettings,
                             Set<String> taskHeaders) {
+        this(settings, transport, threadPool, transportInterceptor, localNodeFactory, clusterSettings, taskHeaders,
+            new ConnectionManager(settings, transport, threadPool));
+    }
+
+    public TransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor transportInterceptor,
+                            Function<BoundTransportAddress, DiscoveryNode> localNodeFactory, @Nullable ClusterSettings clusterSettings,
+                            Set<String> taskHeaders, ConnectionManager connectionManager) {
         super(settings);
         // The only time we do not want to validate node connections is when this is a transport client using the simple node sampler
         this.validateConnections = TransportClient.CLIENT_TYPE.equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey())) == false ||
@@ -152,6 +171,7 @@ public TransportService(Settings settings, Transport transport, ThreadPool threa
         this.transport = transport;
         this.threadPool = threadPool;
         this.localNodeFactory = localNodeFactory;
+        this.connectionManager = connectionManager;
         this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         setTracerLogInclude(TRACE_LOG_INCLUDE_SETTING.get(settings));
         setTracerLogExclude(TRACE_LOG_EXCLUDE_SETTING.get(settings));
@@ -162,6 +182,7 @@ public TransportService(Settings settings, Transport transport, ThreadPool threa
         this.connectToRemoteCluster = RemoteClusterService.ENABLE_REMOTE_CLUSTERS.get(settings);
         remoteClusterService = new RemoteClusterService(settings, this);
         responseHandlers = transport.getResponseHandlers();
+        defaultConnectionProfile = TcpTransport.buildDefaultConnectionProfile(settings);
         if (clusterSettings != null) {
             clusterSettings.addSettingsUpdateConsumer(TRACE_LOG_INCLUDE_SETTING, this::setTracerLogInclude);
             clusterSettings.addSettingsUpdateConsumer(TRACE_LOG_EXCLUDE_SETTING, this::setTracerLogExclude);
@@ -232,6 +253,7 @@ protected void doStart() {
     @Override
     protected void doStop() {
         try {
+            connectionManager.close();
             transport.stop();
         } finally {
             // in case the transport is not connected to our local node (thus cleaned on node disconnect)
@@ -305,7 +327,7 @@ public List<String> getLocalAddresses() {
      * Returns <code>true</code> iff the given node is already connected.
      */
     public boolean nodeConnected(DiscoveryNode node) {
-        return isLocalNode(node) || transport.nodeConnected(node);
+        return isLocalNode(node) || connectionManager.nodeConnected(node);
     }
 
     /**
@@ -327,7 +349,9 @@ public void connectToNode(final DiscoveryNode node, ConnectionProfile connection
         if (isLocalNode(node)) {
             return;
         }
-        transport.connectToNode(node, connectionProfile, (newConnection, actualProfile) -> {
+
+        ConnectionProfile resolvedProfile = TcpTransport.resolveConnectionProfile(connectionProfile, defaultConnectionProfile);
+        connectionManager.connectToNode(node, resolvedProfile, (newConnection, actualProfile) -> {
             // We don't validate cluster names to allow for CCS connections.
             final DiscoveryNode remote = handshake(newConnection, actualProfile.getHandshakeTimeout().millis(), cn -> true).discoveryNode;
             if (validateConnections && node.equals(remote) == false) {
@@ -378,12 +402,11 @@ public DiscoveryNode handshake(
      * @param handshakeTimeout handshake timeout
      * @param clusterNamePredicate cluster name validation predicate
      * @return the handshake response
-     * @throws ConnectTransportException if the connection failed
      * @throws IllegalStateException if the handshake failed
      */
     public HandshakeResponse handshake(
         final Transport.Connection connection,
-        final long handshakeTimeout, Predicate<ClusterName> clusterNamePredicate) throws ConnectTransportException {
+        final long handshakeTimeout, Predicate<ClusterName> clusterNamePredicate) {
         final HandshakeResponse response;
         final DiscoveryNode node = connection.getNode();
         try {
@@ -410,6 +433,10 @@ public HandshakeResponse newInstance() {
         return response;
     }
 
+    public ConnectionManager getConnectionManager() {
+        return connectionManager;
+    }
+
     static class HandshakeRequest extends TransportRequest {
 
         public static final HandshakeRequest INSTANCE = new HandshakeRequest();
@@ -462,15 +489,17 @@ public void disconnectFromNode(DiscoveryNode node) {
         if (isLocalNode(node)) {
             return;
         }
-        transport.disconnectFromNode(node);
+        connectionManager.disconnectFromNode(node);
     }
 
     public void addConnectionListener(TransportConnectionListener listener) {
         transport.addConnectionListener(listener);
+        connectionManager.addListener(listener);
     }
 
     public void removeConnectionListener(TransportConnectionListener listener) {
         transport.removeConnectionListener(listener);
+        connectionManager.removeListener(listener);
     }
 
     public <T extends TransportResponse> TransportFuture<T> submitRequest(DiscoveryNode node, String action, TransportRequest request,
@@ -533,7 +562,7 @@ public Transport.Connection getConnection(DiscoveryNode node) {
         if (isLocalNode(node)) {
             return localNodeConnection;
         } else {
-            return transport.getConnection(node);
+            return connectionManager.getConnection(node);
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java b/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java
index 5a36b3b5e8583..1d2a4ca6d5f75 100644
--- a/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java
@@ -20,15 +20,27 @@
 package org.elasticsearch;
 
 import org.apache.commons.codec.DecoderException;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.RemoteClusterAware;
 
 import java.util.Optional;
 
 import static org.elasticsearch.ExceptionsHelper.MAX_ITERATIONS;
 import static org.elasticsearch.ExceptionsHelper.maybeError;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.nullValue;
 
 public class ExceptionsHelperTests extends ESTestCase {
 
@@ -91,4 +103,84 @@ public void testStatus() {
         assertThat(ExceptionsHelper.status(new EsRejectedExecutionException("rejected")), equalTo(RestStatus.TOO_MANY_REQUESTS));
     }
 
+    public void testGroupBy() {
+        ShardOperationFailedException[] failures = new ShardOperationFailedException[]{
+            createShardFailureParsingException("error", "node0", "index", 0, null),
+            createShardFailureParsingException("error", "node1", "index", 1, null),
+            createShardFailureParsingException("error", "node2", "index2", 2, null),
+            createShardFailureParsingException("error", "node0", "index", 0, "cluster1"),
+            createShardFailureParsingException("error", "node1", "index", 1, "cluster1"),
+            createShardFailureParsingException("error", "node2", "index", 2, "cluster1"),
+            createShardFailureParsingException("error", "node0", "index", 0, "cluster2"),
+            createShardFailureParsingException("error", "node1", "index", 1, "cluster2"),
+            createShardFailureParsingException("error", "node2", "index", 2, "cluster2"),
+            createShardFailureParsingException("another error", "node2", "index", 2, "cluster2")
+        };
+
+        ShardOperationFailedException[] groupBy = ExceptionsHelper.groupBy(failures);
+        assertThat(groupBy.length, equalTo(5));
+        String[] expectedIndices = new String[]{"index", "index2", "cluster1:index", "cluster2:index", "cluster2:index"};
+        String[] expectedErrors = new String[]{"error", "error", "error", "error", "another error"};
+        int i = 0;
+        for (ShardOperationFailedException shardOperationFailedException : groupBy) {
+            assertThat(shardOperationFailedException.getCause().getMessage(), equalTo(expectedErrors[i]));
+            assertThat(shardOperationFailedException.index(), equalTo(expectedIndices[i++]));
+        }
+    }
+
+    private static ShardSearchFailure createShardFailureParsingException(String error, String nodeId,
+                                                                         String index, int shardId, String clusterAlias) {
+        ParsingException ex = new ParsingException(0, 0, error, new IllegalArgumentException("some bad argument"));
+        ex.setIndex(index);
+        return new ShardSearchFailure(ex, createSearchShardTarget(nodeId, shardId, index, clusterAlias));
+    }
+
+    private static SearchShardTarget createSearchShardTarget(String nodeId, int shardId, String index, String clusterAlias) {
+        return new SearchShardTarget(nodeId,
+            new ShardId(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), clusterAlias, OriginalIndices.NONE);
+    }
+
+    public void testGroupByNullTarget() {
+        ShardOperationFailedException[] failures = new ShardOperationFailedException[] {
+            createShardFailureQueryShardException("error", "index", null),
+            createShardFailureQueryShardException("error", "index", null),
+            createShardFailureQueryShardException("error", "index", null),
+            createShardFailureQueryShardException("error", "index", "cluster1"),
+            createShardFailureQueryShardException("error", "index", "cluster1"),
+            createShardFailureQueryShardException("error", "index", "cluster1"),
+            createShardFailureQueryShardException("error", "index", "cluster2"),
+            createShardFailureQueryShardException("error", "index", "cluster2"),
+            createShardFailureQueryShardException("error", "index2", null),
+            createShardFailureQueryShardException("another error", "index2", null),
+        };
+
+        ShardOperationFailedException[] groupBy = ExceptionsHelper.groupBy(failures);
+        assertThat(groupBy.length, equalTo(5));
+        String[] expectedIndices = new String[]{"index", "cluster1:index", "cluster2:index", "index2", "index2"};
+        String[] expectedErrors = new String[]{"error", "error", "error", "error", "another error"};
+        int i = 0;
+        for (ShardOperationFailedException shardOperationFailedException : groupBy) {
+            assertThat(shardOperationFailedException.index(), nullValue());
+            assertThat(shardOperationFailedException.getCause(), instanceOf(ElasticsearchException.class));
+            ElasticsearchException elasticsearchException = (ElasticsearchException) shardOperationFailedException.getCause();
+            assertThat(elasticsearchException.getMessage(), equalTo(expectedErrors[i]));
+            assertThat(elasticsearchException.getIndex().getName(), equalTo(expectedIndices[i++]));
+        }
+    }
+
+    private static ShardSearchFailure createShardFailureQueryShardException(String error, String indexName, String clusterAlias) {
+        Index index = new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexName), "uuid");
+        QueryShardException queryShardException = new QueryShardException(index, error, new IllegalArgumentException("parse error"));
+        return new ShardSearchFailure(queryShardException, null);
+    }
+
+    public void testGroupByNullIndex() {
+        ShardOperationFailedException[] failures = new ShardOperationFailedException[] {
+            new ShardSearchFailure(new IllegalArgumentException("error")),
+            new ShardSearchFailure(new ParsingException(0, 0, "error", null)),
+        };
+
+        ShardOperationFailedException[] groupBy = ExceptionsHelper.groupBy(failures);
+        assertThat(groupBy.length, equalTo(2));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/ShardOperationFailedExceptionTests.java b/server/src/test/java/org/elasticsearch/action/ShardOperationFailedExceptionTests.java
new file mode 100644
index 0000000000000..1348445b62752
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/ShardOperationFailedExceptionTests.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class ShardOperationFailedExceptionTests extends ESTestCase {
+
+    public void testCauseCannotBeNull() {
+        NullPointerException nullPointerException = expectThrows(NullPointerException.class, () -> new Failure(
+            randomAlphaOfLengthBetween(3, 10), randomInt(), randomAlphaOfLengthBetween(5, 10), randomFrom(RestStatus.values()), null));
+        assertEquals("cause cannot be null", nullPointerException.getMessage());
+    }
+
+    public void testStatusCannotBeNull() {
+        NullPointerException nullPointerException = expectThrows(NullPointerException.class, () -> new Failure(
+            randomAlphaOfLengthBetween(3, 10), randomInt(), randomAlphaOfLengthBetween(5, 10), null, new IllegalArgumentException()));
+        assertEquals("status cannot be null", nullPointerException.getMessage());
+    }
+
+    public void testReasonCannotBeNull() {
+        NullPointerException nullPointerException = expectThrows(NullPointerException.class, () -> new Failure(
+            randomAlphaOfLengthBetween(3, 10), randomInt(), null, randomFrom(RestStatus.values()), new IllegalArgumentException()));
+        assertEquals("reason cannot be null", nullPointerException.getMessage());
+    }
+
+    public void testIndexIsNullable() {
+        new Failure(null, randomInt(), randomAlphaOfLengthBetween(5, 10), randomFrom(RestStatus.values()), new IllegalArgumentException());
+    }
+
+    private static class Failure extends ShardOperationFailedException {
+
+        Failure(@Nullable String index, int shardId, String reason, RestStatus status, Throwable cause) {
+           super(index, shardId, reason, status, cause);
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return null;
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
index c8503603f665c..7952758240544 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
-import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
@@ -205,14 +204,7 @@ public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
                             assertThat(nodesMap.size(), equalTo(cluster().size()));
                             for (final NodesReloadSecureSettingsResponse.NodeResponse nodeResponse : nodesReloadResponse.getNodes()) {
                                 assertThat(nodeResponse.reloadException(), notNullValue());
-                                // Running in a JVM with a BouncyCastle FIPS Security Provider, decrypting the Keystore with the wrong
-                                // password returns a SecurityException if the DataInputStream can't be fully consumed
-                                if (inFipsJvm()) {
                                     assertThat(nodeResponse.reloadException(), instanceOf(SecurityException.class));
-                                } else {
-                                    assertThat(nodeResponse.reloadException(), instanceOf(IOException.class));
-                                }
-
                             }
                         } catch (final AssertionError e) {
                             reloadSettingsError.set(e);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java
index 02a98eacda0e9..b67c2e2954d04 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java
@@ -63,7 +63,7 @@ public void setUp() throws Exception {
         clusterService = getInstanceFromNode(ClusterService.class);
         indicesService = getInstanceFromNode(IndicesService.class);
         CapturingTransport capturingTransport = new CapturingTransport();
-        transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool,
+        transportService = capturingTransport.createCapturingTransportService(clusterService.getSettings(), threadPool,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR,
             boundAddress -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
@@ -81,12 +81,10 @@ public void tearDown() throws Exception {
     public void testIncludeDefaults() {
         GetIndexRequest defaultsRequest = new GetIndexRequest().indices(indexName).includeDefaults(true);
         getIndexAction.execute(null, defaultsRequest, ActionListener.wrap(
-            defaultsResponse -> {
-                assertNotNull(
-                    "index.refresh_interval should be set as we are including defaults",
-                    defaultsResponse.getSetting(indexName, "index.refresh_interval")
-                );
-            }, exception -> {
+            defaultsResponse -> assertNotNull(
+                "index.refresh_interval should be set as we are including defaults",
+                defaultsResponse.getSetting(indexName, "index.refresh_interval")
+            ), exception -> {
                 throw new AssertionError(exception);
             })
         );
@@ -95,12 +93,10 @@ public void testIncludeDefaults() {
     public void testDoNotIncludeDefaults() {
         GetIndexRequest noDefaultsRequest = new GetIndexRequest().indices(indexName);
         getIndexAction.execute(null, noDefaultsRequest, ActionListener.wrap(
-            noDefaultsResponse -> {
-                assertNull(
-                    "index.refresh_interval should be null as it was never set",
-                    noDefaultsResponse.getSetting(indexName, "index.refresh_interval")
-                );
-            }, exception -> {
+            noDefaultsResponse -> assertNull(
+                "index.refresh_interval should be null as it was never set",
+                noDefaultsResponse.getSetting(indexName, "index.refresh_interval")
+            ), exception -> {
                 throw new AssertionError(exception);
             })
         );
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index 5379769e819dc..7823cd8849cf7 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -60,7 +60,12 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
 
 
     public void testRolloverOnEmptyIndex() throws Exception {
-        assertAcked(prepareCreate("test_index-1").addAlias(new Alias("test_alias")).get());
+        Alias testAlias = new Alias("test_alias");
+        boolean explicitWriteIndex = randomBoolean();
+        if (explicitWriteIndex) {
+            testAlias.writeIndex(true);
+        }
+        assertAcked(prepareCreate("test_index-1").addAlias(testAlias).get());
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
         assertThat(response.getOldIndex(), equalTo("test_index-1"));
         assertThat(response.getNewIndex(), equalTo("test_index-000002"));
@@ -69,7 +74,12 @@ public void testRolloverOnEmptyIndex() throws Exception {
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index-1");
-        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        if (explicitWriteIndex) {
+            assertTrue(oldIndex.getAliases().containsKey("test_alias"));
+            assertFalse(oldIndex.getAliases().get("test_alias").writeIndex());
+        } else {
+            assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        }
         final IndexMetaData newIndex = state.metaData().index("test_index-000002");
         assertTrue(newIndex.getAliases().containsKey("test_alias"));
     }
@@ -97,8 +107,49 @@ public void testRollover() throws Exception {
             is(both(greaterThanOrEqualTo(beforeTime)).and(lessThanOrEqualTo(client().threadPool().absoluteTimeInMillis() + 1000L))));
     }
 
+    public void testRolloverWithExplicitWriteIndex() throws Exception {
+        long beforeTime = client().threadPool().absoluteTimeInMillis() - 1000L;
+        assertAcked(prepareCreate("test_index-2").addAlias(new Alias("test_alias").writeIndex(true)).get());
+        index("test_index-2", "type1", "1", "field", "value");
+        flush("test_index-2");
+        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
+        assertThat(response.getOldIndex(), equalTo("test_index-2"));
+        assertThat(response.getNewIndex(), equalTo("test_index-000003"));
+        assertThat(response.isDryRun(), equalTo(false));
+        assertThat(response.isRolledOver(), equalTo(true));
+        assertThat(response.getConditionStatus().size(), equalTo(0));
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetaData oldIndex = state.metaData().index("test_index-2");
+        assertTrue(oldIndex.getAliases().containsKey("test_alias"));
+        assertFalse(oldIndex.getAliases().get("test_alias").writeIndex());
+        final IndexMetaData newIndex = state.metaData().index("test_index-000003");
+        assertTrue(newIndex.getAliases().containsKey("test_alias"));
+        assertTrue(newIndex.getAliases().get("test_alias").writeIndex());
+        assertThat(oldIndex.getRolloverInfos().size(), equalTo(1));
+        assertThat(oldIndex.getRolloverInfos().get("test_alias").getAlias(), equalTo("test_alias"));
+        assertThat(oldIndex.getRolloverInfos().get("test_alias").getMetConditions(), is(empty()));
+        assertThat(oldIndex.getRolloverInfos().get("test_alias").getTime(),
+            is(both(greaterThanOrEqualTo(beforeTime)).and(lessThanOrEqualTo(client().threadPool().absoluteTimeInMillis() + 1000L))));
+    }
+
+    public void testRolloverWithNoWriteIndex() {
+        Boolean firstIsWriteIndex = randomFrom(false, null);
+        assertAcked(prepareCreate("index1").addAlias(new Alias("alias").writeIndex(firstIsWriteIndex)).get());
+        if (firstIsWriteIndex == null) {
+            assertAcked(prepareCreate("index2").addAlias(new Alias("alias").writeIndex(randomFrom(false, null))).get());
+        }
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> client().admin().indices().prepareRolloverIndex("alias").dryRun(randomBoolean()).get());
+        assertThat(exception.getMessage(), equalTo("source alias [alias] does not point to a write index"));
+    }
+
     public void testRolloverWithIndexSettings() throws Exception {
-        assertAcked(prepareCreate("test_index-2").addAlias(new Alias("test_alias")).get());
+        Alias testAlias = new Alias("test_alias");
+        boolean explicitWriteIndex = randomBoolean();
+        if (explicitWriteIndex) {
+            testAlias.writeIndex(true);
+        }
+        assertAcked(prepareCreate("test_index-2").addAlias(testAlias).get());
         index("test_index-2", "type1", "1", "field", "value");
         flush("test_index-2");
         final Settings settings = Settings.builder()
@@ -114,12 +165,17 @@ public void testRolloverWithIndexSettings() throws Exception {
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index-2");
-        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
         final IndexMetaData newIndex = state.metaData().index("test_index-000003");
         assertThat(newIndex.getNumberOfShards(), equalTo(1));
         assertThat(newIndex.getNumberOfReplicas(), equalTo(0));
         assertTrue(newIndex.getAliases().containsKey("test_alias"));
         assertTrue(newIndex.getAliases().containsKey("extra_alias"));
+        if (explicitWriteIndex) {
+            assertFalse(oldIndex.getAliases().get("test_alias").writeIndex());
+            assertTrue(newIndex.getAliases().get("test_alias").writeIndex());
+        } else {
+            assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        }
     }
 
     public void testRolloverDryRun() throws Exception {
@@ -140,7 +196,12 @@ public void testRolloverDryRun() throws Exception {
     }
 
     public void testRolloverConditionsNotMet() throws Exception {
-        assertAcked(prepareCreate("test_index-0").addAlias(new Alias("test_alias")).get());
+        boolean explicitWriteIndex = randomBoolean();
+        Alias testAlias = new Alias("test_alias");
+        if (explicitWriteIndex) {
+            testAlias.writeIndex(true);
+        }
+        assertAcked(prepareCreate("test_index-0").addAlias(testAlias).get());
         index("test_index-0", "type1", "1", "field", "value");
         flush("test_index-0");
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
@@ -160,12 +221,22 @@ public void testRolloverConditionsNotMet() throws Exception {
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index-0");
         assertTrue(oldIndex.getAliases().containsKey("test_alias"));
+        if (explicitWriteIndex) {
+            assertTrue(oldIndex.getAliases().get("test_alias").writeIndex());
+        } else {
+            assertNull(oldIndex.getAliases().get("test_alias").writeIndex());
+        }
         final IndexMetaData newIndex = state.metaData().index("test_index-000001");
         assertNull(newIndex);
     }
 
     public void testRolloverWithNewIndexName() throws Exception {
-        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
+        Alias testAlias = new Alias("test_alias");
+        boolean explicitWriteIndex = randomBoolean();
+        if (explicitWriteIndex) {
+            testAlias.writeIndex(true);
+        }
+        assertAcked(prepareCreate("test_index").addAlias(testAlias).get());
         index("test_index", "type1", "1", "field", "value");
         flush("test_index");
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
@@ -177,9 +248,14 @@ public void testRolloverWithNewIndexName() throws Exception {
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index");
-        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
         final IndexMetaData newIndex = state.metaData().index("test_new_index");
         assertTrue(newIndex.getAliases().containsKey("test_alias"));
+        if (explicitWriteIndex) {
+            assertFalse(oldIndex.getAliases().get("test_alias").writeIndex());
+            assertTrue(newIndex.getAliases().get("test_alias").writeIndex());
+        } else {
+            assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        }
     }
 
     public void testRolloverOnExistingIndex() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 6149c380cd737..d59700f2b7a45 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -172,39 +172,75 @@ public void testCreateUpdateAliasRequest() {
         assertTrue(foundRemove);
     }
 
+    public void testCreateUpdateAliasRequestWithExplicitWriteIndex() {
+        String sourceAlias = randomAlphaOfLength(10);
+        String sourceIndex = randomAlphaOfLength(10);
+        String targetIndex = randomAlphaOfLength(10);
+        final RolloverRequest rolloverRequest = new RolloverRequest(sourceAlias, targetIndex);
+        final IndicesAliasesClusterStateUpdateRequest updateRequest =
+            TransportRolloverAction.prepareRolloverAliasesWriteIndexUpdateRequest(sourceIndex, targetIndex, rolloverRequest);
+
+        List<AliasAction> actions = updateRequest.actions();
+        assertThat(actions, hasSize(2));
+        boolean foundAddWrite = false;
+        boolean foundRemoveWrite = false;
+        for (AliasAction action : actions) {
+            AliasAction.Add addAction = (AliasAction.Add) action;
+            if (action.getIndex().equals(targetIndex)) {
+                assertEquals(sourceAlias, addAction.getAlias());
+                assertTrue(addAction.writeIndex());
+                foundAddWrite = true;
+            } else if (action.getIndex().equals(sourceIndex)) {
+                assertEquals(sourceAlias, addAction.getAlias());
+                assertFalse(addAction.writeIndex());
+                foundRemoveWrite = true;
+            } else {
+                throw new AssertionError("Unknow index [" + action.getIndex() + "]");
+            }
+        }
+        assertTrue(foundAddWrite);
+        assertTrue(foundRemoveWrite);
+    }
+
     public void testValidation() {
         String index1 = randomAlphaOfLength(10);
-        String alias = randomAlphaOfLength(10);
+        String aliasWithWriteIndex = randomAlphaOfLength(10);
         String index2 = randomAlphaOfLength(10);
-        String aliasWithMultipleIndices = randomAlphaOfLength(10);
+        String aliasWithNoWriteIndex = randomAlphaOfLength(10);
+        Boolean firstIsWriteIndex = randomFrom(false, null);
         final Settings settings = Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
             .build();
-        final MetaData metaData = MetaData.builder()
+        MetaData.Builder metaDataBuilder = MetaData.builder()
             .put(IndexMetaData.builder(index1)
                 .settings(settings)
-                .putAlias(AliasMetaData.builder(alias))
-                .putAlias(AliasMetaData.builder(aliasWithMultipleIndices))
-            )
-            .put(IndexMetaData.builder(index2)
-                .settings(settings)
-                .putAlias(AliasMetaData.builder(aliasWithMultipleIndices))
-            ).build();
+                .putAlias(AliasMetaData.builder(aliasWithWriteIndex))
+                .putAlias(AliasMetaData.builder(aliasWithNoWriteIndex).writeIndex(firstIsWriteIndex))
+            );
+        IndexMetaData.Builder indexTwoBuilder = IndexMetaData.builder(index2).settings(settings);
+        if (firstIsWriteIndex == null) {
+            indexTwoBuilder.putAlias(AliasMetaData.builder(aliasWithNoWriteIndex).writeIndex(randomFrom(false, null)));
+        }
+        metaDataBuilder.put(indexTwoBuilder);
+        MetaData metaData = metaDataBuilder.build();
 
-        expectThrows(IllegalArgumentException.class, () ->
-            TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithMultipleIndices,
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+            TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithNoWriteIndex,
                 randomAlphaOfLength(10))));
-        expectThrows(IllegalArgumentException.class, () ->
+        assertThat(exception.getMessage(), equalTo("source alias [" + aliasWithNoWriteIndex + "] does not point to a write index"));
+        exception = expectThrows(IllegalArgumentException.class, () ->
             TransportRolloverAction.validate(metaData, new RolloverRequest(randomFrom(index1, index2),
                 randomAlphaOfLength(10))));
-        expectThrows(IllegalArgumentException.class, () ->
+        assertThat(exception.getMessage(), equalTo("source alias is a concrete index"));
+        exception = expectThrows(IllegalArgumentException.class, () ->
             TransportRolloverAction.validate(metaData, new RolloverRequest(randomAlphaOfLength(5),
                 randomAlphaOfLength(10)))
         );
-        TransportRolloverAction.validate(metaData, new RolloverRequest(alias, randomAlphaOfLength(10)));
+        assertThat(exception.getMessage(), equalTo("source alias does not exist"));
+        TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithWriteIndex, randomAlphaOfLength(10)));
     }
 
     public void testGenerateRolloverIndexName() {
@@ -248,7 +284,7 @@ public void testCreateIndexRequest() {
     public void testRejectDuplicateAlias() {
         final IndexTemplateMetaData template = IndexTemplateMetaData.builder("test-template")
             .patterns(Arrays.asList("foo-*", "bar-*"))
-            .putAlias(AliasMetaData.builder("foo-write")).putAlias(AliasMetaData.builder("bar-write"))
+            .putAlias(AliasMetaData.builder("foo-write")).putAlias(AliasMetaData.builder("bar-write").writeIndex(randomBoolean()))
             .build();
         final MetaData metaData = MetaData.builder().put(createMetaData(), false).put(template).build();
         String indexName = randomFrom("foo-123", "bar-xyz");
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java
index 11f0188c8c0b0..03ccebba10dbd 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java
@@ -75,7 +75,7 @@ public void setUp() throws Exception {
         threadPool = new TestThreadPool("GetSettingsActionTests");
         clusterService = createClusterService(threadPool);
         CapturingTransport capturingTransport = new CapturingTransport();
-        transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool,
+        transportService = capturingTransport.createCapturingTransportService(clusterService.getSettings(), threadPool,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR,
             boundAddress -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestStreamableTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestStreamableTests.java
index e8dd3943cb762..83ddb45655182 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestStreamableTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestStreamableTests.java
@@ -41,7 +41,7 @@ protected UpdateSettingsRequest mutateInstance(UpdateSettingsRequest request) {
         List<Runnable> mutators = new ArrayList<>();
         mutators.add(() -> mutation
                 .masterNodeTimeout(randomValueOtherThan(request.masterNodeTimeout().getStringRep(), ESTestCase::randomTimeValue)));
-        mutators.add(() -> mutation.timeout(randomValueOtherThan(request.masterNodeTimeout().getStringRep(), ESTestCase::randomTimeValue)));
+        mutators.add(() -> mutation.timeout(randomValueOtherThan(request.timeout().getStringRep(), ESTestCase::randomTimeValue)));
         mutators.add(() -> mutation.settings(mutateSettings(request.settings())));
         mutators.add(() -> mutation.indices(mutateIndices(request.indices())));
         mutators.add(() -> mutation.indicesOptions(randomValueOtherThan(request.indicesOptions(),
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContextTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContextTests.java
new file mode 100644
index 0000000000000..de7444fac0903
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContextTests.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.bulk.TransportShardBulkActionTests.FakeDeleteResult;
+import org.elasticsearch.action.bulk.TransportShardBulkActionTests.FakeIndexResult;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class BulkPrimaryExecutionContextTests extends ESTestCase {
+
+    public void testAbortedSkipped() {
+        BulkShardRequest shardRequest = generateRandomRequest();
+
+        ArrayList<DocWriteRequest<?>> nonAbortedRequests = new ArrayList<>();
+        for (BulkItemRequest request : shardRequest.items()) {
+            if (randomBoolean()) {
+                request.abort("index", new ElasticsearchException("bla"));
+            } else {
+                nonAbortedRequests.add(request.request());
+            }
+        }
+
+        ArrayList<DocWriteRequest<?>> visitedRequests = new ArrayList<>();
+        for (BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(shardRequest, null);
+             context.hasMoreOperationsToExecute();
+             ) {
+            visitedRequests.add(context.getCurrent());
+            context.setRequestToExecute(context.getCurrent());
+            // using failures prevents caring about types
+            context.markOperationAsExecuted(new Engine.IndexResult(new ElasticsearchException("bla"), 1, 1));
+            context.markAsCompleted(context.getExecutionResult());
+        }
+
+        assertThat(visitedRequests, equalTo(nonAbortedRequests));
+    }
+
+    private BulkShardRequest generateRandomRequest() {
+        BulkItemRequest[] items = new BulkItemRequest[randomInt(20)];
+        for (int i = 0; i < items.length; i++) {
+            final DocWriteRequest request;
+            switch (randomFrom(DocWriteRequest.OpType.values())) {
+                case INDEX:
+                    request = new IndexRequest("index", "_doc", "id_" + i);
+                    break;
+                case CREATE:
+                    request = new IndexRequest("index", "_doc", "id_" + i).create(true);
+                    break;
+                case UPDATE:
+                    request = new UpdateRequest("index", "_doc", "id_" + i);
+                    break;
+                case DELETE:
+                    request = new DeleteRequest("index", "_doc", "id_" + i);
+                    break;
+                default:
+                    throw new AssertionError("unknown type");
+            }
+            items[i] = new BulkItemRequest(i, request);
+        }
+        return new BulkShardRequest(new ShardId("index", "_na_", 0),
+            randomFrom(WriteRequest.RefreshPolicy.values()), items);
+    }
+
+    public void testTranslogLocation() {
+
+        BulkShardRequest shardRequest = generateRandomRequest();
+
+        Translog.Location expectedLocation = null;
+        final IndexShard primary = mock(IndexShard.class);
+        when(primary.shardId()).thenReturn(shardRequest.shardId());
+
+        long translogGen = 0;
+        long translogOffset = 0;
+
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(shardRequest, primary);
+        while (context.hasMoreOperationsToExecute()) {
+            final Engine.Result result;
+            final DocWriteRequest<?> current = context.getCurrent();
+            final boolean failure = rarely();
+            if (frequently()) {
+                translogGen += randomIntBetween(1, 4);
+                translogOffset = 0;
+            } else {
+                translogOffset += randomIntBetween(200, 400);
+            }
+
+            Translog.Location location = new Translog.Location(translogGen, translogOffset, randomInt(200));
+            switch (current.opType()) {
+                case INDEX:
+                case CREATE:
+                    context.setRequestToExecute(current);
+                    if (failure) {
+                        result = new Engine.IndexResult(new ElasticsearchException("bla"), 1, 1);
+                    } else {
+                        result = new FakeIndexResult(1, 1, randomLongBetween(0, 200), randomBoolean(), location);
+                    }
+                    break;
+                case UPDATE:
+                    context.setRequestToExecute(new IndexRequest(current.index(), current.type(), current.id()));
+                    if (failure) {
+                        result = new Engine.IndexResult(new ElasticsearchException("bla"), 1, 1, 1);
+                    } else {
+                        result = new FakeIndexResult(1, 1, randomLongBetween(0, 200), randomBoolean(), location);
+                    }
+                    break;
+                case DELETE:
+                    context.setRequestToExecute(current);
+                    if (failure) {
+                        result = new Engine.DeleteResult(new ElasticsearchException("bla"), 1, 1);
+                    } else {
+                        result = new FakeDeleteResult(1, 1, randomLongBetween(0, 200), randomBoolean(), location);
+                    }
+                    break;
+                default:
+                    throw new AssertionError("unknown type:" + current.opType());
+            }
+            if (failure == false) {
+                expectedLocation = location;
+            }
+            context.markOperationAsExecuted(result);
+            context.markAsCompleted(context.getExecutionResult());
+        }
+
+        assertThat(context.getLocationToSync(), equalTo(expectedLocation));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java
index ca82e7f12227d..8a7c46ebcf60e 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java
@@ -402,19 +402,23 @@ public void testBulkUpdateLargerVolume() throws Exception {
                     .setScript(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "ctx.op = \"delete\"", Collections.emptyMap())));
         }
         response = builder.execute().actionGet();
-        assertThat(response.hasFailures(), equalTo(false));
+        assertThat("expected no failures but got: " + response.buildFailureMessage(), response.hasFailures(), equalTo(false));
         assertThat(response.getItems().length, equalTo(numDocs));
         for (int i = 0; i < numDocs; i++) {
-            assertThat(response.getItems()[i].getItemId(), equalTo(i));
-            assertThat(response.getItems()[i].getId(), equalTo(Integer.toString(i)));
-            assertThat(response.getItems()[i].getIndex(), equalTo("test"));
-            assertThat(response.getItems()[i].getType(), equalTo("type1"));
-            assertThat(response.getItems()[i].getOpType(), equalTo(OpType.UPDATE));
+            final BulkItemResponse itemResponse = response.getItems()[i];
+            assertThat(itemResponse.getFailure(), nullValue());
+            assertThat(itemResponse.isFailed(), equalTo(false));
+            assertThat(itemResponse.getItemId(), equalTo(i));
+            assertThat(itemResponse.getId(), equalTo(Integer.toString(i)));
+            assertThat(itemResponse.getIndex(), equalTo("test"));
+            assertThat(itemResponse.getType(), equalTo("type1"));
+            assertThat(itemResponse.getOpType(), equalTo(OpType.UPDATE));
             for (int j = 0; j < 5; j++) {
                 GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(i)).get();
                 assertThat(getResponse.isExists(), equalTo(false));
             }
         }
+        assertThat(response.hasFailures(), equalTo(false));
     }
 
     public void testBulkIndexingWhileInitializing() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
index 1d6b77fc747c7..1a0e314d88680 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -45,6 +46,7 @@
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class TransportBulkActionIndicesThatCannotBeCreatedTests extends ESTestCase {
     public void testNonExceptional() {
@@ -97,7 +99,11 @@ public void testSomeFail() {
 
     private void indicesThatCannotBeCreatedTestCase(Set<String> expected,
             BulkRequest bulkRequest, Function<String, Boolean> shouldAutoCreate) {
-        TransportBulkAction action = new TransportBulkAction(Settings.EMPTY, null, mock(TransportService.class), mock(ClusterService.class),
+        ClusterService clusterService = mock(ClusterService.class);
+        ClusterState state = mock(ClusterState.class);
+        when(state.getMetaData()).thenReturn(MetaData.EMPTY_META_DATA);
+        when(clusterService.state()).thenReturn(state);
+        TransportBulkAction action = new TransportBulkAction(Settings.EMPTY, null, mock(TransportService.class), clusterService,
                 null, null, null, mock(ActionFilters.class), null, null) {
             @Override
             void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener<BulkResponse> listener,
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
index 66527726573a5..4c0dacc8a6e73 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
@@ -19,22 +19,31 @@
 
 package org.elasticsearch.action.bulk;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateApplier;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.ingest.PipelineExecutionService;
 import org.elasticsearch.tasks.Task;
@@ -68,6 +77,14 @@
 
 public class TransportBulkActionIngestTests extends ESTestCase {
 
+    /**
+     * Index for which mock settings contain a default pipeline.
+     */
+    private static final String WITH_DEFAULT_PIPELINE = "index_with_default_pipeline";
+
+    private static final Settings SETTINGS =
+        Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), true).build();
+
     /** Services needed by bulk action */
     TransportService transportService;
     ClusterService clusterService;
@@ -103,25 +120,42 @@ public class TransportBulkActionIngestTests extends ESTestCase {
     /** A subclass of the real bulk action to allow skipping real bulk indexing, and marking when it would have happened. */
     class TestTransportBulkAction extends TransportBulkAction {
         boolean isExecuted = false; // set when the "real" bulk execution happens
+
+        boolean needToCheck; // pluggable return value for `needToCheck`
+
+        boolean indexCreated = true; // If set to false, will be set to true by call to createIndex
+
         TestTransportBulkAction() {
-            super(Settings.EMPTY, null, transportService, clusterService, ingestService,
-                null, null, new ActionFilters(Collections.emptySet()), null, null);
+            super(SETTINGS, null, transportService, clusterService, ingestService,
+                null, null, new ActionFilters(Collections.emptySet()), null,
+                new AutoCreateIndex(
+                    SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                    new IndexNameExpressionResolver(SETTINGS)
+                )
+            );
         }
         @Override
         protected boolean needToCheck() {
-            return false;
+            return needToCheck;
         }
         @Override
         void executeBulk(Task task, final BulkRequest bulkRequest, final long startTimeNanos, final ActionListener<BulkResponse> listener,
                 final AtomicArray<BulkItemResponse> responses, Map<String, IndexNotFoundException> indicesThatCannotBeCreated) {
+            assertTrue(indexCreated);
             isExecuted = true;
         }
+
+        @Override
+        void createIndex(String index, TimeValue timeout, ActionListener<CreateIndexResponse> listener) {
+            indexCreated = true;
+            listener.onResponse(null);
+        }
     }
 
     class TestSingleItemBulkWriteAction extends TransportSingleItemBulkWriteAction<IndexRequest, IndexResponse> {
 
         TestSingleItemBulkWriteAction(TestTransportBulkAction bulkAction) {
-            super(Settings.EMPTY, IndexAction.NAME, TransportBulkActionIngestTests.this.transportService,
+            super(SETTINGS, IndexAction.NAME, TransportBulkActionIngestTests.this.transportService,
                     TransportBulkActionIngestTests.this.clusterService,
                     null, null, null, new ActionFilters(Collections.emptySet()), null,
                     IndexRequest::new, IndexRequest::new, ThreadPool.Names.WRITE, bulkAction, null);
@@ -153,6 +187,17 @@ public void setupAction() {
         when(nodes.getIngestNodes()).thenReturn(ingestNodes);
         ClusterState state = mock(ClusterState.class);
         when(state.getNodes()).thenReturn(nodes);
+        MetaData metaData = MetaData.builder().indices(ImmutableOpenMap.<String, IndexMetaData>builder()
+            .putAll(
+                Collections.singletonMap(
+                    WITH_DEFAULT_PIPELINE,
+                    IndexMetaData.builder(WITH_DEFAULT_PIPELINE).settings(
+                        settings(Version.CURRENT).put(IndexSettings.DEFAULT_PIPELINE.getKey(), "default_pipeline")
+                            .build()
+                    ).numberOfShards(1).numberOfReplicas(1).build()))
+            .build()).build();
+        when(state.getMetaData()).thenReturn(metaData);
+        when(state.metaData()).thenReturn(metaData);
         when(clusterService.state()).thenReturn(state);
         doAnswer(invocation -> {
             ClusterChangedEvent event = mock(ClusterChangedEvent.class);
@@ -227,7 +272,7 @@ public void testIngestLocal() throws Exception {
         // now check success
         Iterator<DocWriteRequest<?>> req = bulkDocsItr.getValue().iterator();
         failureHandler.getValue().accept((IndexRequest)req.next(), exception); // have an exception for our one index request
-        indexRequest2.setPipeline(null); // this is done by the real pipeline execution service when processing
+        indexRequest2.setPipeline(IngestService.NOOP_PIPELINE_NAME); // this is done by the real pipeline execution service when processing
         completionHandler.getValue().accept(null);
         assertTrue(action.isExecuted);
         assertFalse(responseCalled.get()); // listener would only be called by real index action, not our mocked one
@@ -259,7 +304,7 @@ public void testSingleItemBulkActionIngestLocal() throws Exception {
         assertTrue(failureCalled.get());
 
         // now check success
-        indexRequest.setPipeline(null); // this is done by the real pipeline execution service when processing
+        indexRequest.setPipeline(IngestService.NOOP_PIPELINE_NAME); // this is done by the real pipeline execution service when processing
         completionHandler.getValue().accept(null);
         assertTrue(action.isExecuted);
         assertFalse(responseCalled.get()); // listener would only be called by real index action, not our mocked one
@@ -359,4 +404,67 @@ public void testSingleItemBulkActionIngestForward() throws Exception {
         }
     }
 
+    public void testUseDefaultPipeline() throws Exception {
+        Exception exception = new Exception("fake exception");
+        IndexRequest indexRequest = new IndexRequest(WITH_DEFAULT_PIPELINE, "type", "id");
+        indexRequest.source(Collections.emptyMap());
+        AtomicBoolean responseCalled = new AtomicBoolean(false);
+        AtomicBoolean failureCalled = new AtomicBoolean(false);
+        singleItemBulkWriteAction.execute(null, indexRequest, ActionListener.wrap(
+            response -> {
+                responseCalled.set(true);
+            },
+            e -> {
+                assertThat(e, sameInstance(exception));
+                failureCalled.set(true);
+            }));
+
+        // check failure works, and passes through to the listener
+        assertFalse(action.isExecuted); // haven't executed yet
+        assertFalse(responseCalled.get());
+        assertFalse(failureCalled.get());
+        verify(executionService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture());
+        completionHandler.getValue().accept(exception);
+        assertTrue(failureCalled.get());
+
+        // now check success
+        indexRequest.setPipeline(IngestService.NOOP_PIPELINE_NAME); // this is done by the real pipeline execution service when processing
+        completionHandler.getValue().accept(null);
+        assertTrue(action.isExecuted);
+        assertFalse(responseCalled.get()); // listener would only be called by real index action, not our mocked one
+        verifyZeroInteractions(transportService);
+    }
+
+    public void testCreateIndexBeforeRunPipeline() throws Exception {
+        Exception exception = new Exception("fake exception");
+        IndexRequest indexRequest = new IndexRequest("missing_index", "type", "id");
+        indexRequest.setPipeline("testpipeline");
+        indexRequest.source(Collections.emptyMap());
+        AtomicBoolean responseCalled = new AtomicBoolean(false);
+        AtomicBoolean failureCalled = new AtomicBoolean(false);
+        action.needToCheck = true;
+        action.indexCreated = false;
+        singleItemBulkWriteAction.execute(null, indexRequest, ActionListener.wrap(
+            response -> responseCalled.set(true),
+            e -> {
+                assertThat(e, sameInstance(exception));
+                failureCalled.set(true);
+            }));
+
+        // check failure works, and passes through to the listener
+        assertFalse(action.isExecuted); // haven't executed yet
+        assertFalse(responseCalled.get());
+        assertFalse(failureCalled.get());
+        verify(executionService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture());
+        completionHandler.getValue().accept(exception);
+        assertTrue(failureCalled.get());
+
+        // now check success
+        indexRequest.setPipeline(IngestService.NOOP_PIPELINE_NAME); // this is done by the real pipeline execution service when processing
+        completionHandler.getValue().accept(null);
+        assertTrue(action.isExecuted);
+        assertFalse(responseCalled.get()); // listener would only be called by real index action, not our mocked one
+        verifyZeroInteractions(transportService);
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
index 3bd66af1bab05..a1abd4d61f7fd 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
@@ -80,8 +80,8 @@ public void setUp() throws Exception {
         threadPool = new TestThreadPool("TransportBulkActionTookTests");
         clusterService = createClusterService(threadPool);
         CapturingTransport capturingTransport = new CapturingTransport();
-        transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+        transportService = capturingTransport.createCapturingTransportService(clusterService.getSettings(), threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR,
             boundAddress -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index 9d5193180299d..f6559f226417e 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -92,8 +92,8 @@ public void tearDown() throws Exception {
 
     private TransportBulkAction createAction(boolean controlled, AtomicLong expected) {
         CapturingTransport capturingTransport = new CapturingTransport();
-        TransportService transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+        TransportService transportService = capturingTransport.createCapturingTransportService(clusterService.getSettings(), threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR,
             boundAddress -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
index 006d2d04fdde4..ca19dcc250948 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
-import org.elasticsearch.action.support.replication.ReplicationOperation;
 import org.elasticsearch.action.support.replication.TransportWriteAction.WritePrimaryResult;
 import org.elasticsearch.action.update.UpdateHelper;
 import org.elasticsearch.action.update.UpdateRequest;
@@ -39,12 +38,13 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Mapping;
+import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardTestCase;
 import org.elasticsearch.index.shard.ShardId;
@@ -52,10 +52,9 @@
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.LongSupplier;
 
 import static org.elasticsearch.action.bulk.TransportShardBulkAction.replicaItemExecutionMode;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -63,103 +62,103 @@
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class TransportShardBulkActionTests extends IndexShardTestCase {
 
     private final ShardId shardId = new ShardId("index", "_na_", 0);
     private final Settings idxSettings = Settings.builder()
-            .put("index.number_of_shards", 1)
-            .put("index.number_of_replicas", 0)
-            .put("index.version.created", Version.CURRENT.id)
-            .build();
+        .put("index.number_of_shards", 1)
+        .put("index.number_of_replicas", 0)
+        .put("index.version.created", Version.CURRENT.id)
+        .build();
 
     private IndexMetaData indexMetaData() throws IOException {
         return IndexMetaData.builder("index")
-                .putMapping("_doc",
-                        "{\"properties\":{\"foo\":{\"type\":\"text\",\"fields\":" +
-                                "{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}")
-                .settings(idxSettings)
-                .primaryTerm(0, 1).build();
+            .putMapping("_doc",
+                "{\"properties\":{\"foo\":{\"type\":\"text\",\"fields\":" +
+                    "{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}")
+            .settings(idxSettings)
+            .primaryTerm(0, 1).build();
     }
 
     public void testShouldExecuteReplicaItem() throws Exception {
         // Successful index request should be replicated
         DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id")
-                .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar");
+            .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar");
         DocWriteResponse response = new IndexResponse(shardId, "type", "id", 1, 17, 1, randomBoolean());
         BulkItemRequest request = new BulkItemRequest(0, writeRequest);
         request.setPrimaryResponse(new BulkItemResponse(0, DocWriteRequest.OpType.INDEX, response));
         assertThat(replicaItemExecutionMode(request, 0),
-                equalTo(ReplicaItemExecutionMode.NORMAL));
+            equalTo(ReplicaItemExecutionMode.NORMAL));
 
         // Failed index requests without sequence no should not be replicated
         writeRequest = new IndexRequest("index", "_doc", "id")
-                .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar");
+            .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar");
         request = new BulkItemRequest(0, writeRequest);
         request.setPrimaryResponse(
-                new BulkItemResponse(0, DocWriteRequest.OpType.INDEX,
-                        new BulkItemResponse.Failure("index", "type", "id",
-                                                     new IllegalArgumentException("i died"))));
+            new BulkItemResponse(0, DocWriteRequest.OpType.INDEX,
+                new BulkItemResponse.Failure("index", "type", "id",
+                    new IllegalArgumentException("i died"))));
         assertThat(replicaItemExecutionMode(request, 0),
-                equalTo(ReplicaItemExecutionMode.NOOP));
+            equalTo(ReplicaItemExecutionMode.NOOP));
 
         // Failed index requests with sequence no should be replicated
         request = new BulkItemRequest(0, writeRequest);
         request.setPrimaryResponse(
-                new BulkItemResponse(0, DocWriteRequest.OpType.INDEX,
-                        new BulkItemResponse.Failure("index", "type", "id",
-                                new IllegalArgumentException(
-                                        "i died after sequence no was generated"),
-                                1)));
+            new BulkItemResponse(0, DocWriteRequest.OpType.INDEX,
+                new BulkItemResponse.Failure("index", "type", "id",
+                    new IllegalArgumentException(
+                        "i died after sequence no was generated"),
+                    1)));
         assertThat(replicaItemExecutionMode(request, 0),
-                equalTo(ReplicaItemExecutionMode.FAILURE));
+            equalTo(ReplicaItemExecutionMode.FAILURE));
         // NOOP requests should not be replicated
         DocWriteRequest<UpdateRequest> updateRequest = new UpdateRequest("index", "type", "id");
         response = new UpdateResponse(shardId, "type", "id", 1, DocWriteResponse.Result.NOOP);
         request = new BulkItemRequest(0, updateRequest);
         request.setPrimaryResponse(new BulkItemResponse(0, DocWriteRequest.OpType.UPDATE,
-                        response));
+            response));
         assertThat(replicaItemExecutionMode(request, 0),
-                equalTo(ReplicaItemExecutionMode.NOOP));
+            equalTo(ReplicaItemExecutionMode.NOOP));
     }
 
-
     public void testExecuteBulkIndexRequest() throws Exception {
-        IndexMetaData metaData = indexMetaData();
         IndexShard shard = newStartedShard(true);
 
         BulkItemRequest[] items = new BulkItemRequest[1];
         boolean create = randomBoolean();
         DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id").source(Requests.INDEX_CONTENT_TYPE)
-                .create(create);
+            .create(create);
         BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
         items[0] = primaryRequest;
         BulkShardRequest bulkShardRequest =
-                new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
+            new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        Translog.Location location = new Translog.Location(0, 0, 0);
         UpdateHelper updateHelper = null;
-
-        Translog.Location newLocation = TransportShardBulkAction.executeBulkItemRequest(metaData,
-                shard, bulkShardRequest, location, 0, updateHelper,
-                threadPool::absoluteTimeInMillis, new NoopMappingUpdatePerformer());
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis,
+            new NoopMappingUpdatePerformer(), () -> {});
+        assertFalse(context.hasMoreOperationsToExecute());
 
         // Translog should change, since there were no problems
-        assertThat(newLocation, not(location));
+        assertNotNull(context.getLocationToSync());
 
         BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
 
         assertThat(primaryResponse.getItemId(), equalTo(0));
         assertThat(primaryResponse.getId(), equalTo("id"));
         assertThat(primaryResponse.getOpType(),
-                equalTo(create ? DocWriteRequest.OpType.CREATE : DocWriteRequest.OpType.INDEX));
+            equalTo(create ? DocWriteRequest.OpType.CREATE : DocWriteRequest.OpType.INDEX));
         assertFalse(primaryResponse.isFailed());
 
         // Assert that the document actually made it there
@@ -170,13 +169,12 @@ public void testExecuteBulkIndexRequest() throws Exception {
         items[0] = primaryRequest;
         bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        Translog.Location secondLocation =
-                TransportShardBulkAction.executeBulkItemRequest( metaData,
-                        shard, bulkShardRequest, newLocation, 0, updateHelper,
-                        threadPool::absoluteTimeInMillis, new ThrowingMappingUpdatePerformer(new RuntimeException("fail")));
+        BulkPrimaryExecutionContext secondContext = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        TransportShardBulkAction.executeBulkItemRequest(secondContext, updateHelper,
+            threadPool::absoluteTimeInMillis, new ThrowingMappingUpdatePerformer(new RuntimeException("fail")), () -> {});
+        assertFalse(context.hasMoreOperationsToExecute());
 
-        // Translog should not change, since the document was not indexed due to a version conflict
-        assertThat(secondLocation, equalTo(newLocation));
+        assertNull(secondContext.getLocationToSync());
 
         BulkItemRequest replicaRequest = bulkShardRequest.items()[0];
 
@@ -194,7 +192,7 @@ public void testExecuteBulkIndexRequest() throws Exception {
         assertThat(failure.getId(), equalTo("id"));
         assertThat(failure.getCause().getClass(), equalTo(VersionConflictEngineException.class));
         assertThat(failure.getCause().getMessage(),
-                containsString("version conflict, document already exists (current version [1])"));
+            containsString("version conflict, document already exists (current version [1])"));
         assertThat(failure.getStatus(), equalTo(RestStatus.CONFLICT));
 
         assertThat(replicaRequest, equalTo(primaryRequest));
@@ -224,7 +222,8 @@ public void testSkipBulkIndexRequestIfAborted() throws Exception {
 
         UpdateHelper updateHelper = null;
         WritePrimaryResult<BulkShardRequest, BulkShardResponse> result = TransportShardBulkAction.performOnPrimary(
-            bulkShardRequest, shard, updateHelper, threadPool::absoluteTimeInMillis, new NoopMappingUpdatePerformer());
+            bulkShardRequest, shard, updateHelper, threadPool::absoluteTimeInMillis, new NoopMappingUpdatePerformer(),
+            () -> {});
 
         // since at least 1 item passed, the tran log location should exist,
         assertThat(result.location, notNullValue());
@@ -254,52 +253,85 @@ public void testSkipBulkIndexRequestIfAborted() throws Exception {
         closeShards(shard);
     }
 
-    public void testExecuteBulkIndexRequestWithRejection() throws Exception {
-        IndexMetaData metaData = indexMetaData();
-        IndexShard shard = newStartedShard(true);
+    public void testExecuteBulkIndexRequestWithMappingUpdates() throws Exception {
 
         BulkItemRequest[] items = new BulkItemRequest[1];
         DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id")
-                .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar");
+            .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar");
         items[0] = new BulkItemRequest(0, writeRequest);
         BulkShardRequest bulkShardRequest =
-                new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
+            new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        Translog.Location location = new Translog.Location(0, 0, 0);
-        UpdateHelper updateHelper = null;
+        Engine.IndexResult mappingUpdate =
+            new Engine.IndexResult(new Mapping(null, null, new MetadataFieldMapper[0], Collections.emptyMap()));
+        Translog.Location resultLocation = new Translog.Location(42, 42, 42);
+        Engine.IndexResult success = new FakeIndexResult(1, 1, 13, true, resultLocation);
+
+        IndexShard shard = mock(IndexShard.class);
+        when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyBoolean())).thenReturn(mappingUpdate);
+
+        // Pretend the mappings haven't made it to the node yet
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        AtomicInteger updateCalled = new AtomicInteger();
+        TransportShardBulkAction.executeBulkItemRequest(context, null, threadPool::absoluteTimeInMillis,
+            (update, shardId, type) -> {
+                // There should indeed be a mapping update
+                assertNotNull(update);
+                updateCalled.incrementAndGet();
+            }, () -> {});
+        assertTrue(context.isInitial());
+        assertTrue(context.hasMoreOperationsToExecute());
+
+        assertThat("mappings were \"updated\" once", updateCalled.get(), equalTo(1));
+
+        // Verify that the shard "executed" the operation twice
+        verify(shard, times(2)).applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyBoolean());
 
-        // Pretend the mappings haven't made it to the node yet, and throw a rejection
-        expectThrows(ReplicationOperation.RetryOnPrimaryException.class,
-            () -> TransportShardBulkAction.executeBulkItemRequest(metaData, shard, bulkShardRequest,
-                location, 0, updateHelper, threadPool::absoluteTimeInMillis,
-                new NoopMappingUpdatePerformer()));
+        when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyBoolean())).thenReturn(success);
+
+        TransportShardBulkAction.executeBulkItemRequest(context, null, threadPool::absoluteTimeInMillis,
+            (update, shardId, type) -> fail("should not have had to update the mappings"), () -> {});
+
+
+        // Verify that the shard "executed" the operation only once (2 for previous invocations plus
+        // 1 for this execution)
+        verify(shard, times(3)).applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyBoolean());
+
+
+        BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
+
+        assertThat(primaryResponse.getItemId(), equalTo(0));
+        assertThat(primaryResponse.getId(), equalTo("id"));
+        assertThat(primaryResponse.getOpType(), equalTo(writeRequest.opType()));
+        assertFalse(primaryResponse.isFailed());
 
         closeShards(shard);
     }
 
     public void testExecuteBulkIndexRequestWithErrorWhileUpdatingMapping() throws Exception {
-        IndexMetaData metaData = indexMetaData();
         IndexShard shard = newStartedShard(true);
 
         BulkItemRequest[] items = new BulkItemRequest[1];
         DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id")
-                .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar");
+            .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar");
         items[0] = new BulkItemRequest(0, writeRequest);
-        BulkShardRequest bulkShardRequest =
-                new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
+        BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        Translog.Location location = new Translog.Location(0, 0, 0);
         UpdateHelper updateHelper = null;
 
-        // Return an exception when trying to update the mapping
+        // Return an exception when trying to update the mapping, or when waiting for it to come
         RuntimeException err = new RuntimeException("some kind of exception");
 
-        Translog.Location newLocation = TransportShardBulkAction.executeBulkItemRequest(metaData,
-                shard, bulkShardRequest, location, 0, updateHelper,
-                threadPool::absoluteTimeInMillis, new ThrowingMappingUpdatePerformer(err));
+        boolean errorOnWait = randomBoolean();
+
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis,
+            errorOnWait == false ? new ThrowingMappingUpdatePerformer(err) : new NoopMappingUpdatePerformer(),
+            errorOnWait ? () -> { throw err; } : () -> {});
+        assertFalse(context.hasMoreOperationsToExecute());
 
-        // Translog shouldn't change, as there were conflicting mappings
-        assertThat(newLocation, equalTo(location));
+        // Translog shouldn't be synced, as there were conflicting mappings
+        assertThat(context.getLocationToSync(), nullValue());
 
         BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
 
@@ -320,24 +352,24 @@ public void testExecuteBulkIndexRequestWithErrorWhileUpdatingMapping() throws Ex
     }
 
     public void testExecuteBulkDeleteRequest() throws Exception {
-        IndexMetaData metaData = indexMetaData();
         IndexShard shard = newStartedShard(true);
 
         BulkItemRequest[] items = new BulkItemRequest[1];
         DocWriteRequest<DeleteRequest> writeRequest = new DeleteRequest("index", "_doc", "id");
         items[0] = new BulkItemRequest(0, writeRequest);
         BulkShardRequest bulkShardRequest =
-                new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
+            new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
         Translog.Location location = new Translog.Location(0, 0, 0);
         UpdateHelper updateHelper = null;
 
-        Translog.Location newLocation = TransportShardBulkAction.executeBulkItemRequest(metaData,
-                shard, bulkShardRequest, location, 0, updateHelper,
-                threadPool::absoluteTimeInMillis, new NoopMappingUpdatePerformer());
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis,
+            new NoopMappingUpdatePerformer(), () -> {});
+        assertFalse(context.hasMoreOperationsToExecute());
 
         // Translog changes, even though the document didn't exist
-        assertThat(newLocation, not(location));
+        assertThat(context.getLocationToSync(), not(location));
 
         BulkItemRequest replicaRequest = bulkShardRequest.items()[0];
         DocWriteRequest<?> replicaDeleteRequest = replicaRequest.request();
@@ -369,14 +401,15 @@ public void testExecuteBulkDeleteRequest() throws Exception {
         items[0] = new BulkItemRequest(0, writeRequest);
         bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        location = newLocation;
+        location = context.getLocationToSync();
 
-        newLocation = TransportShardBulkAction.executeBulkItemRequest(metaData, shard,
-                bulkShardRequest, location, 0, updateHelper, threadPool::absoluteTimeInMillis,
-                new NoopMappingUpdatePerformer());
+        context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis,
+            new NoopMappingUpdatePerformer(), () -> {});
+        assertFalse(context.hasMoreOperationsToExecute());
 
         // Translog changes, because the document was deleted
-        assertThat(newLocation, not(location));
+        assertThat(context.getLocationToSync(), not(location));
 
         replicaRequest = bulkShardRequest.items()[0];
         replicaDeleteRequest = replicaRequest.request();
@@ -405,63 +438,81 @@ public void testExecuteBulkDeleteRequest() throws Exception {
         closeShards(shard);
     }
 
-    public void testNoopUpdateReplicaRequest() throws Exception {
-        DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id")
-                .source(Requests.INDEX_CONTENT_TYPE, "field", "value");
-        BulkItemRequest replicaRequest = new BulkItemRequest(0, writeRequest);
+    public void testNoopUpdateRequest() throws Exception {
+        DocWriteRequest<UpdateRequest> writeRequest = new UpdateRequest("index", "_doc", "id")
+            .doc(Requests.INDEX_CONTENT_TYPE, "field", "value");
+        BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
 
         DocWriteResponse noopUpdateResponse = new UpdateResponse(shardId, "_doc", "id", 0,
-                DocWriteResponse.Result.NOOP);
-        BulkItemResultHolder noopResults = new BulkItemResultHolder(noopUpdateResponse, null,
-                replicaRequest);
+            DocWriteResponse.Result.NOOP);
 
-        Translog.Location location = new Translog.Location(0, 0, 0);
-        BulkItemRequest[] items = new BulkItemRequest[0];
+        IndexShard shard = mock(IndexShard.class);
+
+        UpdateHelper updateHelper = mock(UpdateHelper.class);
+        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+            new UpdateHelper.Result(noopUpdateResponse, DocWriteResponse.Result.NOOP,
+                Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE));
+
+        BulkItemRequest[] items = new BulkItemRequest[]{primaryRequest};
         BulkShardRequest bulkShardRequest =
-                new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
+            new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        BulkItemResponse primaryResponse = TransportShardBulkAction.createPrimaryResponse(
-                noopResults, DocWriteRequest.OpType.UPDATE, bulkShardRequest);
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis,
+            new NoopMappingUpdatePerformer(), () -> {});
 
-        Translog.Location newLocation =
-                TransportShardBulkAction.calculateTranslogLocation(location, noopResults);
+        assertFalse(context.hasMoreOperationsToExecute());
 
         // Basically nothing changes in the request since it's a noop
-        assertThat(newLocation, equalTo(location));
+        assertThat(context.getLocationToSync(), nullValue());
+        BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
         assertThat(primaryResponse.getItemId(), equalTo(0));
         assertThat(primaryResponse.getId(), equalTo("id"));
         assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
         assertThat(primaryResponse.getResponse(), equalTo(noopUpdateResponse));
         assertThat(primaryResponse.getResponse().getResult(),
-                equalTo(DocWriteResponse.Result.NOOP));
+            equalTo(DocWriteResponse.Result.NOOP));
+        assertThat(bulkShardRequest.items().length, equalTo(1));
+        assertEquals(primaryRequest, bulkShardRequest.items()[0]); // check that bulk item was not mutated
+        assertThat(primaryResponse.getResponse().getSeqNo(), equalTo(SequenceNumbers.UNASSIGNED_SEQ_NO));
     }
 
-    public void testUpdateReplicaRequestWithFailure() throws Exception {
-        DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id").source(Requests.INDEX_CONTENT_TYPE);
-        BulkItemRequest replicaRequest = new BulkItemRequest(0, writeRequest);
+    public void testUpdateRequestWithFailure() throws Exception {
+        IndexSettings indexSettings = new IndexSettings(indexMetaData(), Settings.EMPTY);
+        DocWriteRequest<UpdateRequest> writeRequest = new UpdateRequest("index", "_doc", "id")
+            .doc(Requests.INDEX_CONTENT_TYPE, "field", "value");
+        BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
+
+        IndexRequest updateResponse = new IndexRequest("index", "_doc", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value");
 
         Exception err = new ElasticsearchException("I'm dead <(x.x)>");
-        Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0);
-        BulkItemResultHolder failedResults = new BulkItemResultHolder(null, indexResult,
-                replicaRequest);
+        Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0, 0);
+        IndexShard shard = mock(IndexShard.class);
+        when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyBoolean())).thenReturn(indexResult);
+        when(shard.indexSettings()).thenReturn(indexSettings);
 
-        Translog.Location location = new Translog.Location(0, 0, 0);
-        BulkItemRequest[] items = new BulkItemRequest[0];
+        UpdateHelper updateHelper = mock(UpdateHelper.class);
+        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+            new UpdateHelper.Result(updateResponse, randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
+                Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE));
+
+        BulkItemRequest[] items = new BulkItemRequest[]{primaryRequest};
         BulkShardRequest bulkShardRequest =
-                new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
-        BulkItemResponse primaryResponse =
-                TransportShardBulkAction.createPrimaryResponse(
-                        failedResults, DocWriteRequest.OpType.UPDATE, bulkShardRequest);
+            new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
+
 
-        Translog.Location newLocation =
-                TransportShardBulkAction.calculateTranslogLocation(location, failedResults);
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis,
+            new NoopMappingUpdatePerformer(), () -> {});
+        assertFalse(context.hasMoreOperationsToExecute());
 
         // Since this was not a conflict failure, the primary response
         // should be filled out with the failure information
-        assertThat(newLocation, equalTo(location));
+        assertNull(context.getLocationToSync());
+        BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
         assertThat(primaryResponse.getItemId(), equalTo(0));
         assertThat(primaryResponse.getId(), equalTo("id"));
-        assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
+        assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
         assertTrue(primaryResponse.isFailed());
         assertThat(primaryResponse.getFailureMessage(), containsString("I'm dead <(x.x)>"));
         BulkItemResponse.Failure failure = primaryResponse.getFailure();
@@ -472,33 +523,42 @@ public void testUpdateReplicaRequestWithFailure() throws Exception {
         assertThat(failure.getStatus(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
     }
 
-    public void testUpdateReplicaRequestWithConflictFailure() throws Exception {
-        DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id").source(Requests.INDEX_CONTENT_TYPE);
-        BulkItemRequest replicaRequest = new BulkItemRequest(0, writeRequest);
 
-        Exception err = new VersionConflictEngineException(shardId, "_doc", "id",
-                "I'm conflicted <(;_;)>");
-        Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0);
-        BulkItemResultHolder failedResults = new BulkItemResultHolder(null, indexResult,
-                replicaRequest);
+    public void testUpdateRequestWithConflictFailure() throws Exception {
+        IndexSettings indexSettings = new IndexSettings(indexMetaData(), Settings.EMPTY);
+        DocWriteRequest<UpdateRequest> writeRequest = new UpdateRequest("index", "_doc", "id")
+            .doc(Requests.INDEX_CONTENT_TYPE, "field", "value");
+        BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
 
-        Translog.Location location = new Translog.Location(0, 0, 0);
-        BulkItemRequest[] items = new BulkItemRequest[0];
+        IndexRequest updateResponse = new IndexRequest("index", "_doc", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value");
+
+        Exception err = new VersionConflictEngineException(shardId, "_doc", "id",
+            "I'm conflicted <(;_;)>");
+        Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0, 0);
+        IndexShard shard = mock(IndexShard.class);
+        when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyBoolean())).thenReturn(indexResult);
+        when(shard.indexSettings()).thenReturn(indexSettings);
+
+        UpdateHelper updateHelper = mock(UpdateHelper.class);
+        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+            new UpdateHelper.Result(updateResponse, randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
+                Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE));
+
+        BulkItemRequest[] items = new BulkItemRequest[]{primaryRequest};
         BulkShardRequest bulkShardRequest =
-                new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
-        BulkItemResponse primaryResponse =
-                TransportShardBulkAction.createPrimaryResponse(
-                        failedResults, DocWriteRequest.OpType.UPDATE, bulkShardRequest);
+            new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        Translog.Location newLocation =
-                TransportShardBulkAction.calculateTranslogLocation(location, failedResults);
 
-        // Since this was not a conflict failure, the primary response
-        // should be filled out with the failure information
-        assertThat(newLocation, equalTo(location));
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis,
+            new NoopMappingUpdatePerformer(), () -> {});
+        assertFalse(context.hasMoreOperationsToExecute());
+
+        assertNull(context.getLocationToSync());
+        BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
         assertThat(primaryResponse.getItemId(), equalTo(0));
         assertThat(primaryResponse.getId(), equalTo("id"));
-        assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
+        assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
         assertTrue(primaryResponse.isFailed());
         assertThat(primaryResponse.getFailureMessage(), containsString("I'm conflicted <(;_;)>"));
         BulkItemResponse.Failure failure = primaryResponse.getFailure();
@@ -509,329 +569,268 @@ public void testUpdateReplicaRequestWithConflictFailure() throws Exception {
         assertThat(failure.getStatus(), equalTo(RestStatus.CONFLICT));
     }
 
-    public void testUpdateReplicaRequestWithSuccess() throws Exception {
-        DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id")
-                .source(Requests.INDEX_CONTENT_TYPE);
-        BulkItemRequest replicaRequest = new BulkItemRequest(0, writeRequest);
+    public void testUpdateRequestWithSuccess() throws Exception {
+        IndexSettings indexSettings = new IndexSettings(indexMetaData(), Settings.EMPTY);
+        DocWriteRequest<UpdateRequest> writeRequest = new UpdateRequest("index", "_doc", "id")
+            .doc(Requests.INDEX_CONTENT_TYPE, "field", "value");
+        BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
+
+        IndexRequest updateResponse = new IndexRequest("index", "_doc", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value");
 
         boolean created = randomBoolean();
         Translog.Location resultLocation = new Translog.Location(42, 42, 42);
-        Engine.IndexResult indexResult = new FakeResult(1, 1, created, resultLocation);
-        DocWriteResponse indexResponse = new IndexResponse(shardId, "_doc", "id", 1, 17, 1, created);
-        BulkItemResultHolder goodResults =
-                new BulkItemResultHolder(indexResponse, indexResult, replicaRequest);
+        Engine.IndexResult indexResult = new FakeIndexResult(1, 1, 13, created, resultLocation);
+        IndexShard shard = mock(IndexShard.class);
+        when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyBoolean())).thenReturn(indexResult);
+        when(shard.indexSettings()).thenReturn(indexSettings);
+
+        UpdateHelper updateHelper = mock(UpdateHelper.class);
+        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+            new UpdateHelper.Result(updateResponse, created ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
+                Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE));
 
-        Translog.Location originalLocation = new Translog.Location(21, 21, 21);
-        BulkItemRequest[] items = new BulkItemRequest[0];
+        BulkItemRequest[] items = new BulkItemRequest[]{primaryRequest};
         BulkShardRequest bulkShardRequest =
-                new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
-        BulkItemResponse primaryResponse =
-                TransportShardBulkAction.createPrimaryResponse(
-                        goodResults, DocWriteRequest.OpType.INDEX, bulkShardRequest);
+            new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        Translog.Location newLocation =
-                TransportShardBulkAction.calculateTranslogLocation(originalLocation, goodResults);
+
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis,
+            new NoopMappingUpdatePerformer(), () -> {});
+        assertFalse(context.hasMoreOperationsToExecute());
 
         // Check that the translog is successfully advanced
-        assertThat(newLocation, equalTo(resultLocation));
+        assertThat(context.getLocationToSync(), equalTo(resultLocation));
+        assertThat(bulkShardRequest.items()[0].request(), equalTo(updateResponse));
         // Since this was not a conflict failure, the primary response
         // should be filled out with the failure information
+        BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
         assertThat(primaryResponse.getItemId(), equalTo(0));
         assertThat(primaryResponse.getId(), equalTo("id"));
-        assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
+        assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
         DocWriteResponse response = primaryResponse.getResponse();
         assertThat(response.status(), equalTo(created ? RestStatus.CREATED : RestStatus.OK));
+        assertThat(response.getSeqNo(), equalTo(13L));
     }
 
-    public void testCalculateTranslogLocation() throws Exception {
-        final Translog.Location original = new Translog.Location(0, 0, 0);
+    public void testUpdateWithDelete() throws Exception {
+        IndexSettings indexSettings = new IndexSettings(indexMetaData(), Settings.EMPTY);
+        DocWriteRequest<UpdateRequest> writeRequest = new UpdateRequest("index", "_doc", "id")
+            .doc(Requests.INDEX_CONTENT_TYPE, "field", "value");
+        BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
 
-        DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id")
-            .source(Requests.INDEX_CONTENT_TYPE);
-        BulkItemRequest replicaRequest = new BulkItemRequest(0, writeRequest);
-        BulkItemResultHolder results = new BulkItemResultHolder(null, null, replicaRequest);
+        DeleteRequest updateResponse = new DeleteRequest("index", "_doc", "id");
 
-        assertThat(TransportShardBulkAction.calculateTranslogLocation(original, results),
-                equalTo(original));
+        boolean found = randomBoolean();
+        Translog.Location resultLocation = new Translog.Location(42, 42, 42);
+        final long resultSeqNo = 13;
+        Engine.DeleteResult deleteResult = new FakeDeleteResult(1, 1, resultSeqNo, found, resultLocation);
+        IndexShard shard = mock(IndexShard.class);
+        when(shard.applyDeleteOperationOnPrimary(anyLong(), any(), any(), any())).thenReturn(deleteResult);
+        when(shard.indexSettings()).thenReturn(indexSettings);
+
+        UpdateHelper updateHelper = mock(UpdateHelper.class);
+        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+            new UpdateHelper.Result(updateResponse, DocWriteResponse.Result.DELETED,
+                Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE));
+
+        BulkItemRequest[] items = new BulkItemRequest[]{primaryRequest};
+        BulkShardRequest bulkShardRequest =
+            new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        boolean created = randomBoolean();
-        DocWriteResponse indexResponse = new IndexResponse(shardId, "_doc", "id", 1, 17, 1, created);
-        Translog.Location newLocation = new Translog.Location(1, 1, 1);
-        final long version = randomNonNegativeLong();
-        final long seqNo = randomNonNegativeLong();
-        Engine.IndexResult indexResult = new IndexResultWithLocation(version, seqNo, created, newLocation);
-        results = new BulkItemResultHolder(indexResponse, indexResult, replicaRequest);
-        assertThat(TransportShardBulkAction.calculateTranslogLocation(original, results),
-                equalTo(newLocation));
 
-    }
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis,
+            new NoopMappingUpdatePerformer(), () -> {});
+        assertFalse(context.hasMoreOperationsToExecute());
 
-    public void testNoOpReplicationOnPrimaryDocumentFailure() throws Exception {
-        final IndexShard shard = spy(newStartedShard(false));
-        BulkItemRequest itemRequest = new BulkItemRequest(0, new IndexRequest("index", "_doc").source(Requests.INDEX_CONTENT_TYPE));
-        final String failureMessage = "simulated primary failure";
-        final IOException exception = new IOException(failureMessage);
-        itemRequest.setPrimaryResponse(new BulkItemResponse(0,
-                randomFrom(
-                        DocWriteRequest.OpType.CREATE,
-                        DocWriteRequest.OpType.DELETE,
-                        DocWriteRequest.OpType.INDEX
-                ),
-                new BulkItemResponse.Failure("index", "_doc", "1",
-                    exception, 1L)
-        ));
-        BulkItemRequest[] itemRequests = new BulkItemRequest[1];
-        itemRequests[0] = itemRequest;
-        BulkShardRequest bulkShardRequest = new BulkShardRequest(
-                shard.shardId(), RefreshPolicy.NONE, itemRequests);
-        TransportShardBulkAction.performOnReplica(bulkShardRequest, shard);
-        verify(shard, times(1)).markSeqNoAsNoop(1, exception.toString());
-        closeShards(shard);
+        // Check that the translog is successfully advanced
+        assertThat(context.getLocationToSync(), equalTo(resultLocation));
+        assertThat(bulkShardRequest.items()[0].request(), equalTo(updateResponse));
+        BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
+        assertThat(primaryResponse.getItemId(), equalTo(0));
+        assertThat(primaryResponse.getId(), equalTo("id"));
+        assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
+        DocWriteResponse response = primaryResponse.getResponse();
+        assertThat(response.status(), equalTo(RestStatus.OK));
+        assertThat(response.getSeqNo(), equalTo(resultSeqNo));
     }
 
-    public void testMappingUpdateParsesCorrectNumberOfTimes() throws Exception {
-        IndexMetaData metaData = indexMetaData();
-        logger.info("--> metadata.getIndex(): {}", metaData.getIndex());
-        final IndexShard shard = spy(newStartedShard(true));
-
-        IndexRequest request = new IndexRequest("index", "_doc", "id")
-                .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar");
-
-        final AtomicInteger updateCalled = new AtomicInteger(0);
-        expectThrows(ReplicationOperation.RetryOnPrimaryException.class,
-            () -> TransportShardBulkAction.executeIndexRequestOnPrimary(request, shard,
-                (update, shardId, type) -> {
-                    // There should indeed be a mapping update
-                    assertNotNull(update);
-                    updateCalled.incrementAndGet();
-                }));
-
-        assertThat("mappings were \"updated\" once", updateCalled.get(), equalTo(1));
 
+    public void testFailureDuringUpdateProcessing() throws Exception {
+        DocWriteRequest<UpdateRequest> writeRequest = new UpdateRequest("index", "_doc", "id")
+            .doc(Requests.INDEX_CONTENT_TYPE, "field", "value");
+        BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
 
-        // Verify that the shard "executed" the operation twice
-        verify(shard, times(2)).applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyBoolean());
+        IndexShard shard = mock(IndexShard.class);
 
-        // Update the mapping, so the next mapping updater doesn't do anything
-        final MapperService mapperService = shard.mapperService();
-        logger.info("--> mapperService.index(): {}", mapperService.index());
-        mapperService.updateMapping(metaData);
+        UpdateHelper updateHelper = mock(UpdateHelper.class);
+        final ElasticsearchException err = new ElasticsearchException("oops");
+        when(updateHelper.prepare(any(), eq(shard), any())).thenThrow(err);
+        BulkItemRequest[] items = new BulkItemRequest[]{primaryRequest};
+        BulkShardRequest bulkShardRequest =
+            new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        TransportShardBulkAction.executeIndexRequestOnPrimary(request, shard,
-            (update, shardId, type) -> fail("should not have had to update the mappings"));
 
-        // Verify that the shard "executed" the operation only once (2 for previous invocations plus
-        // 1 for this execution)
-        verify(shard, times(3)).applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyBoolean());
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis,
+            new NoopMappingUpdatePerformer(), () -> {});
+        assertFalse(context.hasMoreOperationsToExecute());
 
-        closeShards(shard);
+        assertNull(context.getLocationToSync());
+        BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
+        assertThat(primaryResponse.getItemId(), equalTo(0));
+        assertThat(primaryResponse.getId(), equalTo("id"));
+        assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
+        assertTrue(primaryResponse.isFailed());
+        assertThat(primaryResponse.getFailureMessage(), containsString("oops"));
+        BulkItemResponse.Failure failure = primaryResponse.getFailure();
+        assertThat(failure.getIndex(), equalTo("index"));
+        assertThat(failure.getType(), equalTo("_doc"));
+        assertThat(failure.getId(), equalTo("id"));
+        assertThat(failure.getCause(), equalTo(err));
+        assertThat(failure.getStatus(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
     }
 
-    public class IndexResultWithLocation extends Engine.IndexResult {
-        private final Translog.Location location;
-        public IndexResultWithLocation(long version, long seqNo, boolean created, Translog.Location newLocation) {
-            super(version, seqNo, created);
-            this.location = newLocation;
+    public void testTranslogPositionToSync() throws Exception {
+        IndexShard shard = newStartedShard(true);
+
+        BulkItemRequest[] items = new BulkItemRequest[randomIntBetween(2, 5)];
+        for (int i = 0; i < items.length; i++) {
+            DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id_" + i)
+                .source(Requests.INDEX_CONTENT_TYPE)
+                .opType(DocWriteRequest.OpType.INDEX);
+            items[i] = new BulkItemRequest(i, writeRequest);
         }
+        BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        @Override
-        public Translog.Location getTranslogLocation() {
-            return this.location;
+        BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
+        while (context.hasMoreOperationsToExecute()) {
+            TransportShardBulkAction.executeBulkItemRequest(context, null, threadPool::absoluteTimeInMillis,
+                new NoopMappingUpdatePerformer(), () -> {});
         }
-    }
 
-    public void testProcessUpdateResponse() throws Exception {
-        IndexShard shard = newStartedShard(false);
+        assertTrue(shard.isSyncNeeded());
 
-        UpdateRequest updateRequest = new UpdateRequest("index", "_doc", "id");
-        BulkItemRequest request = new BulkItemRequest(0, updateRequest);
-        Exception err = new VersionConflictEngineException(shardId, "_doc", "id",
-                "I'm conflicted <(;_;)>");
-        Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0);
-        Engine.DeleteResult deleteResult = new Engine.DeleteResult(1, 1, true);
-        DocWriteResponse.Result docWriteResult = DocWriteResponse.Result.CREATED;
-        DocWriteResponse.Result deleteWriteResult = DocWriteResponse.Result.DELETED;
-        IndexRequest indexRequest = new IndexRequest("index", "_doc", "id");
-        DeleteRequest deleteRequest = new DeleteRequest("index", "_doc", "id");
-        UpdateHelper.Result translate = new UpdateHelper.Result(indexRequest, docWriteResult,
-                new HashMap<String, Object>(), XContentType.JSON);
-        UpdateHelper.Result translateDelete = new UpdateHelper.Result(deleteRequest, deleteWriteResult,
-                new HashMap<String, Object>(), XContentType.JSON);
+        // if we sync the location, nothing else is unsynced
+        CountDownLatch latch = new CountDownLatch(1);
+        shard.sync(context.getLocationToSync(), e -> {
+            if (e != null) {
+                throw new AssertionError(e);
+            }
+            latch.countDown();
+        });
 
-        BulkItemRequest[] itemRequests = new BulkItemRequest[1];
-        itemRequests[0] = request;
-
-        BulkItemResultHolder holder = TransportShardBulkAction.processUpdateResponse(updateRequest,
-                "index", indexResult, translate, shard, 7);
-
-        assertTrue(holder.isVersionConflict());
-        assertThat(holder.response, instanceOf(UpdateResponse.class));
-        UpdateResponse updateResp = (UpdateResponse) holder.response;
-        assertThat(updateResp.getGetResult(), equalTo(null));
-        assertThat(holder.operationResult, equalTo(indexResult));
-        BulkItemRequest replicaBulkRequest = holder.replicaRequest;
-        assertThat(replicaBulkRequest.id(), equalTo(7));
-        DocWriteRequest<?> replicaRequest = replicaBulkRequest.request();
-        assertThat(replicaRequest, instanceOf(IndexRequest.class));
-        assertThat(replicaRequest, equalTo(indexRequest));
-
-        BulkItemResultHolder deleteHolder = TransportShardBulkAction.processUpdateResponse(updateRequest,
-                "index", deleteResult, translateDelete, shard, 8);
-
-        assertFalse(deleteHolder.isVersionConflict());
-        assertThat(deleteHolder.response, instanceOf(UpdateResponse.class));
-        UpdateResponse delUpdateResp = (UpdateResponse) deleteHolder.response;
-        assertThat(delUpdateResp.getGetResult(), equalTo(null));
-        assertThat(deleteHolder.operationResult, equalTo(deleteResult));
-        BulkItemRequest delReplicaBulkRequest = deleteHolder.replicaRequest;
-        assertThat(delReplicaBulkRequest.id(), equalTo(8));
-        DocWriteRequest<?> delReplicaRequest = delReplicaBulkRequest.request();
-        assertThat(delReplicaRequest, instanceOf(DeleteRequest.class));
-        assertThat(delReplicaRequest, equalTo(deleteRequest));
+        latch.await();
+        assertFalse(shard.isSyncNeeded());
 
         closeShards(shard);
     }
 
-    public void testExecuteUpdateRequestOnce() throws Exception {
-        IndexMetaData metaData = indexMetaData();
-        IndexShard shard = newStartedShard(true);
+    public void testNoOpReplicationOnPrimaryDocumentFailure() throws Exception {
+        final IndexShard shard = spy(newStartedShard(false));
+        BulkItemRequest itemRequest = new BulkItemRequest(0, new IndexRequest("index", "_doc").source(Requests.INDEX_CONTENT_TYPE));
+        final String failureMessage = "simulated primary failure";
+        final IOException exception = new IOException(failureMessage);
+        itemRequest.setPrimaryResponse(new BulkItemResponse(0,
+            randomFrom(
+                DocWriteRequest.OpType.CREATE,
+                DocWriteRequest.OpType.DELETE,
+                DocWriteRequest.OpType.INDEX
+            ),
+            new BulkItemResponse.Failure("index", "_doc", "1",
+                exception, 1L)
+        ));
+        BulkItemRequest[] itemRequests = new BulkItemRequest[1];
+        itemRequests[0] = itemRequest;
+        BulkShardRequest bulkShardRequest = new BulkShardRequest(
+            shard.shardId(), RefreshPolicy.NONE, itemRequests);
+        TransportShardBulkAction.performOnReplica(bulkShardRequest, shard);
+        verify(shard, times(1)).markSeqNoAsNoop(1, exception.toString());
+        closeShards(shard);
+    }
 
-        Map<String, Object> source = new HashMap<>();
-        BulkItemRequest[] items = new BulkItemRequest[1];
-        boolean create = randomBoolean();
-        DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id").source(Requests.INDEX_CONTENT_TYPE)
-                .create(create);
+    public void testRetries() throws Exception {
+        IndexSettings indexSettings = new IndexSettings(indexMetaData(), Settings.EMPTY);
+        UpdateRequest writeRequest = new UpdateRequest("index", "_doc", "id")
+            .doc(Requests.INDEX_CONTENT_TYPE, "field", "value");
+        // the beating will continue until success has come.
+        writeRequest.retryOnConflict(Integer.MAX_VALUE);
         BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
-        items[0] = primaryRequest;
 
-        IndexRequest indexRequest = new IndexRequest("index", "_doc", "id");
-        indexRequest.source(source);
-
-        DocWriteResponse.Result docWriteResult = DocWriteResponse.Result.CREATED;
-        UpdateHelper.Result translate = new UpdateHelper.Result(indexRequest, docWriteResult,
-                new HashMap<String, Object>(), XContentType.JSON);
-        UpdateHelper updateHelper = new MockUpdateHelper(translate);
-        UpdateRequest updateRequest = new UpdateRequest("index", "_doc", "id");
-        updateRequest.upsert(source);
-
-        BulkItemResultHolder holder = TransportShardBulkAction.executeUpdateRequestOnce(updateRequest, shard, metaData,
-                "index", updateHelper, threadPool::absoluteTimeInMillis, primaryRequest, 0,
-            new ThrowingMappingUpdatePerformer(new RuntimeException()));
-
-        assertFalse(holder.isVersionConflict());
-        assertNotNull(holder.response);
-        assertNotNull(holder.operationResult);
-        assertNotNull(holder.replicaRequest);
-
-        assertThat(holder.response, instanceOf(UpdateResponse.class));
-        UpdateResponse updateResp = (UpdateResponse) holder.response;
-        assertThat(updateResp.getGetResult(), equalTo(null));
-        BulkItemRequest replicaBulkRequest = holder.replicaRequest;
-        assertThat(replicaBulkRequest.id(), equalTo(0));
-        DocWriteRequest<?> replicaRequest = replicaBulkRequest.request();
-        assertThat(replicaRequest, instanceOf(IndexRequest.class));
-        assertThat(replicaRequest, equalTo(indexRequest));
+        IndexRequest updateResponse = new IndexRequest("index", "_doc", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value");
 
-        // Assert that the document actually made it there
-        assertDocCount(shard, 1);
-        closeShards(shard);
-    }
-
-    public void testExecuteUpdateRequestOnceWithFailure() throws Exception {
-        IndexMetaData metaData = indexMetaData();
-        IndexShard shard = newStartedShard(true);
+        Exception err = new VersionConflictEngineException(shardId, "_doc", "id",
+            "I'm conflicted <(;_;)>");
+        Engine.IndexResult conflictedResult = new Engine.IndexResult(err, 0, 0);
+        Engine.IndexResult mappingUpdate =
+            new Engine.IndexResult(new Mapping(null, null, new MetadataFieldMapper[0], Collections.emptyMap()));
+        Translog.Location resultLocation = new Translog.Location(42, 42, 42);
+        Engine.IndexResult success = new FakeIndexResult(1, 1, 13, true, resultLocation);
 
-        Map<String, Object> source = new HashMap<>();
-        source.put("foo", "bar");
-        BulkItemRequest[] items = new BulkItemRequest[1];
-        boolean create = randomBoolean();
-        DocWriteRequest<IndexRequest> writeRequest = new IndexRequest("index", "_doc", "id")
-                .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar")
-                .create(create);
-        BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
-        items[0] = primaryRequest;
+        IndexShard shard = mock(IndexShard.class);
+        when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyBoolean())).thenAnswer(ir -> {
+            if (randomBoolean()) {
+                return conflictedResult;
+            }
+            if (randomBoolean()) {
+                return mappingUpdate;
+            } else {
+                return success;
+            }
+        });
+        when(shard.indexSettings()).thenReturn(indexSettings);
 
-        IndexRequest indexRequest = new IndexRequest("index", "_doc", "id");
-        indexRequest.source(source);
-
-        Exception prepareFailure = new IllegalArgumentException("I failed to do something!");
-        UpdateHelper updateHelper = new FailingUpdateHelper(prepareFailure);
-        UpdateRequest updateRequest = new UpdateRequest("index", "_doc", "id");
-        updateRequest.upsert(source);
-
-        BulkItemResultHolder holder = TransportShardBulkAction.executeUpdateRequestOnce(updateRequest, shard, metaData,
-                "index", updateHelper, threadPool::absoluteTimeInMillis, primaryRequest, 0, new NoopMappingUpdatePerformer());
-
-        assertFalse(holder.isVersionConflict());
-        assertNull(holder.response);
-        assertNotNull(holder.operationResult);
-        assertNotNull(holder.replicaRequest);
-
-        Engine.IndexResult opResult = (Engine.IndexResult) holder.operationResult;
-        assertThat(opResult.getResultType(), equalTo(Engine.Result.Type.FAILURE));
-        assertFalse(opResult.isCreated());
-        Exception e = opResult.getFailure();
-        assertThat(e.getMessage(), containsString("I failed to do something!"));
-
-        BulkItemRequest replicaBulkRequest = holder.replicaRequest;
-        assertThat(replicaBulkRequest.id(), equalTo(0));
-        assertThat(replicaBulkRequest.request(), instanceOf(IndexRequest.class));
-        IndexRequest replicaRequest = (IndexRequest) replicaBulkRequest.request();
-        assertThat(replicaRequest.index(), equalTo("index"));
-        assertThat(replicaRequest.type(), equalTo("_doc"));
-        assertThat(replicaRequest.id(), equalTo("id"));
-        assertThat(replicaRequest.sourceAsMap(), equalTo(source));
-
-        // Assert that the document did not make it there, since it should have failed
-        assertDocCount(shard, 0);
-        closeShards(shard);
-    }
+        UpdateHelper updateHelper = mock(UpdateHelper.class);
+        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+            new UpdateHelper.Result(updateResponse, randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
+                Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE));
 
-    /**
-     * Fake UpdateHelper that always returns whatever result you give it
-     */
-    private static class MockUpdateHelper extends UpdateHelper {
-        private final UpdateHelper.Result result;
+        BulkItemRequest[] items = new BulkItemRequest[]{primaryRequest};
+        BulkShardRequest bulkShardRequest =
+            new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
-        MockUpdateHelper(UpdateHelper.Result result) {
-            super(Settings.EMPTY, null);
-            this.result = result;
-        }
+        WritePrimaryResult<BulkShardRequest, BulkShardResponse> result = TransportShardBulkAction.performOnPrimary(
+            bulkShardRequest, shard, updateHelper, threadPool::absoluteTimeInMillis, new NoopMappingUpdatePerformer(),
+            () -> {});
 
-        @Override
-        public UpdateHelper.Result prepare(UpdateRequest u, IndexShard s, LongSupplier n) {
-            logger.info("--> preparing update for {} - {}", s, u);
-            return result;
-        }
+        assertThat(result.location, equalTo(resultLocation));
+        BulkItemResponse primaryResponse = result.replicaRequest().items()[0].getPrimaryResponse();
+        assertThat(primaryResponse.getItemId(), equalTo(0));
+        assertThat(primaryResponse.getId(), equalTo("id"));
+        assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
+        DocWriteResponse response = primaryResponse.getResponse();
+        assertThat(response.status(), equalTo(RestStatus.CREATED));
+        assertThat(response.getSeqNo(), equalTo(13L));
     }
 
     /**
-     * An update helper that always fails to prepare the update
+     * Fake IndexResult that has a settable translog location
      */
-    private static class FailingUpdateHelper extends UpdateHelper {
-        private final Exception e;
+    static class FakeIndexResult extends Engine.IndexResult {
+
+        private final Translog.Location location;
 
-        FailingUpdateHelper(Exception failure) {
-            super(Settings.EMPTY, null);
-            this.e = failure;
+        protected FakeIndexResult(long version, long term, long seqNo, boolean created, Translog.Location location) {
+            super(version, term, seqNo, created);
+            this.location = location;
         }
 
         @Override
-        public UpdateHelper.Result prepare(UpdateRequest u, IndexShard s, LongSupplier n) {
-            logger.info("--> preparing failing update for {} - {}", s, u);
-            throw new ElasticsearchException(e);
+        public Translog.Location getTranslogLocation() {
+            return this.location;
         }
     }
 
     /**
-     * Fake IndexResult that has a settable translog location
+     * Fake DeleteResult that has a settable translog location
      */
-    private static class FakeResult extends Engine.IndexResult {
+    static class FakeDeleteResult extends Engine.DeleteResult {
 
         private final Translog.Location location;
 
-        protected FakeResult(long version, long seqNo, boolean created, Translog.Location location) {
-            super(version, seqNo, created);
+        protected FakeDeleteResult(long version, long term, long seqNo, boolean found, Translog.Location location) {
+            super(version, term, seqNo, found);
             this.location = location;
         }
 
@@ -851,6 +850,7 @@ public void updateMappings(Mapping update, ShardId shardId, String type) {
     /** Always throw the given exception */
     private class ThrowingMappingUpdatePerformer implements MappingUpdatePerformer {
         private final RuntimeException e;
+
         ThrowingMappingUpdatePerformer(RuntimeException e) {
             this.e = e;
         }
diff --git a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
index 640d6a84e960d..3d87b8bd0bed9 100644
--- a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
@@ -199,4 +199,14 @@ public void testToStringSizeLimit() throws UnsupportedEncodingException {
         assertEquals("index {[index][type][null], source[n/a, actual length: [" + new ByteSizeValue(actualBytes).toString() +
                 "], max length: " + new ByteSizeValue(IndexRequest.MAX_SOURCE_LENGTH_IN_TOSTRING).toString() + "]}", request.toString());
     }
+
+    public void testRejectsEmptyStringPipeline() {
+        IndexRequest request = new IndexRequest("index", "type");
+        request.source("{}", XContentType.JSON);
+        request.setPipeline("");
+        ActionRequestValidationException validate = request.validate();
+        assertThat(validate, notNullValue());
+        assertThat(validate.getMessage(),
+            containsString("pipeline cannot be an empty string"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java b/server/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java
deleted file mode 100644
index f68545c3f7921..0000000000000
--- a/server/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.ingest;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
-
-import java.io.IOException;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-
-public class WritePipelineResponseTests extends AbstractStreamableXContentTestCase<WritePipelineResponse> {
-
-    public void testSerializationWithoutError() throws IOException {
-        boolean isAcknowledged = randomBoolean();
-        WritePipelineResponse response;
-        response = new WritePipelineResponse(isAcknowledged);
-        BytesStreamOutput out = new BytesStreamOutput();
-        response.writeTo(out);
-        StreamInput streamInput = out.bytes().streamInput();
-        WritePipelineResponse otherResponse = new WritePipelineResponse();
-        otherResponse.readFrom(streamInput);
-
-        assertThat(otherResponse.isAcknowledged(), equalTo(response.isAcknowledged()));
-    }
-
-    public void testSerializationWithError() throws IOException {
-        WritePipelineResponse response = new WritePipelineResponse();
-        BytesStreamOutput out = new BytesStreamOutput();
-        response.writeTo(out);
-        StreamInput streamInput = out.bytes().streamInput();
-        WritePipelineResponse otherResponse = new WritePipelineResponse();
-        otherResponse.readFrom(streamInput);
-
-        assertThat(otherResponse.isAcknowledged(), equalTo(response.isAcknowledged()));
-    }
-
-    public void testToXContent() {
-        WritePipelineResponse response = new WritePipelineResponse(true);
-        String output = Strings.toString(response);
-        assertEquals("{\"acknowledged\":true}", output);
-    }
-
-    @Override
-    protected WritePipelineResponse doParseInstance(XContentParser parser) {
-        return WritePipelineResponse.fromXContent(parser);
-    }
-
-    @Override
-    protected WritePipelineResponse createTestInstance() {
-        return new WritePipelineResponse(randomBoolean());
-    }
-
-    @Override
-    protected WritePipelineResponse createBlankInstance() {
-        return new WritePipelineResponse();
-    }
-
-    @Override
-    protected WritePipelineResponse mutateInstance(WritePipelineResponse response) {
-        return new WritePipelineResponse(response.isAcknowledged() == false);
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
index 2726de237582d..1f7f6f4249b0c 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -256,7 +256,6 @@ public void run() {
         assertEquals(10, numRequests.get());
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29242")
     public void testFanOutAndCollect() throws InterruptedException {
         SearchRequest request = new SearchRequest();
         request.allowPartialSearchResults(true);
@@ -347,6 +346,9 @@ public void run() throws IOException {
                             sendReleaseSearchContext(result.getRequestId(), new MockConnection(result.node), OriginalIndices.NONE);
                         }
                         responseListener.onResponse(response);
+                        if (latch.getCount() == 0) {
+                            throw new AssertionError("Running a search phase after the latch has reached 0 !!!!");
+                        }
                         latch.countDown();
                     }
                 };
@@ -444,7 +446,17 @@ public void sendRequest(long requestId, String action, TransportRequest request,
         }
 
         @Override
-        public void close() throws IOException {
+        public void addCloseListener(ActionListener<Void> listener) {
+
+        }
+
+        @Override
+        public boolean isClosed() {
+            return false;
+        }
+
+        @Override
+        public void close() {
             throw new UnsupportedOperationException();
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java b/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java
index 13625a2bc612f..bd892829c9548 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java
@@ -45,8 +45,9 @@ public static ShardSearchFailure createTestItem() {
         if (randomBoolean()) {
             String nodeId = randomAlphaOfLengthBetween(5, 10);
             String indexName = randomAlphaOfLengthBetween(5, 10);
+            String clusterAlias = randomBoolean() ? randomAlphaOfLengthBetween(5, 10) : null;
             searchShardTarget = new SearchShardTarget(nodeId,
-                    new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), randomInt()), null, null);
+                    new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), randomInt()), clusterAlias, OriginalIndices.NONE);
         }
         return new ShardSearchFailure(ex, searchShardTarget);
     }
@@ -115,4 +116,22 @@ public void testToXContent() throws IOException {
                 + "}",
                 xContent.utf8ToString());
     }
+
+    public void testToXContentWithClusterAlias() throws IOException {
+        ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(0, 0, "some message", null),
+            new SearchShardTarget("nodeId", new ShardId(new Index("indexName", "indexUuid"), 123), "cluster1", OriginalIndices.NONE));
+        BytesReference xContent = toXContent(failure, XContentType.JSON, randomBoolean());
+        assertEquals(
+            "{\"shard\":123,"
+                + "\"index\":\"cluster1:indexName\","
+                + "\"node\":\"nodeId\","
+                + "\"reason\":{"
+                + "\"type\":\"parsing_exception\","
+                + "\"reason\":\"some message\","
+                + "\"line\":0,"
+                + "\"col\":0"
+                + "}"
+                + "}",
+            xContent.utf8ToString());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index fdc3d890363ad..d12c3ca26eb08 100644
--- a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -191,7 +191,7 @@ public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        final TransportService transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
+        TransportService transportService = transport.createCapturingTransportService(clusterService.getSettings(), THREAD_POOL,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
diff --git a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index b27bc9ad79432..f592b3d803af5 100644
--- a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -87,8 +87,8 @@ public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
+        transportService = transport.createCapturingTransportService(clusterService.getSettings(), threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
         localNode = new DiscoveryNode("local_node", buildNewFakeTransportAddress(), Collections.emptyMap(),
diff --git a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index 8a79da044b915..be7ec477ff48c 100644
--- a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -180,7 +180,7 @@ public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
+        transportService = transport.createCapturingTransportService(clusterService.getSettings(), THREAD_POOL,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index 59ede535c2f39..60053748d68c9 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -27,10 +27,12 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable.Builder;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -44,6 +46,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@@ -93,7 +96,8 @@ public static ClusterState state(String index, boolean activePrimaryLocal, Shard
         IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(Settings.builder()
                 .put(SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas)
-                .put(SETTING_CREATION_DATE, System.currentTimeMillis())).primaryTerm(0, primaryTerm).build();
+                .put(SETTING_CREATION_DATE, System.currentTimeMillis())).primaryTerm(0, primaryTerm)
+            .build();
 
         RoutingTable.Builder routing = new RoutingTable.Builder();
         routing.addAsNew(indexMetaData);
@@ -138,12 +142,19 @@ public static ClusterState state(String index, boolean activePrimaryLocal, Shard
                     TestShardRouting.newShardRouting(index, shardId.id(), replicaNode, relocatingNode, false, replicaState,
                             unassignedInfo));
         }
+        final IndexShardRoutingTable indexShardRoutingTable = indexShardRoutingBuilder.build();
+
+        IndexMetaData.Builder indexMetaDataBuilder = new IndexMetaData.Builder(indexMetaData);
+        indexMetaDataBuilder.putInSyncAllocationIds(0,
+            indexShardRoutingTable.activeShards().stream().map(ShardRouting::allocationId).map(AllocationId::getId)
+            .collect(Collectors.toSet())
+        );
 
         ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
         state.nodes(discoBuilder);
-        state.metaData(MetaData.builder().put(indexMetaData, false).generateClusterUuidIfNeeded());
+        state.metaData(MetaData.builder().put(indexMetaDataBuilder.build(), false).generateClusterUuidIfNeeded());
         state.routingTable(RoutingTable.builder().add(IndexRoutingTable.builder(indexMetaData.getIndex())
-                .addIndexShard(indexShardRoutingBuilder.build())).build());
+                .addIndexShard(indexShardRoutingTable)).build());
         return state.build();
     }
 
@@ -272,21 +283,21 @@ public static ClusterState stateWithAssignedPrimariesAndOneReplica(String index,
         state.routingTable(RoutingTable.builder().add(indexRoutingTableBuilder.build()).build());
         return state.build();
     }
-    
-    
+
+
     /**
      * Creates cluster state with several indexes, shards and replicas and all shards STARTED.
      */
     public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indices, int numberOfShards, int numberOfReplicas) {
 
-        int numberOfDataNodes = numberOfReplicas + 1; 
+        int numberOfDataNodes = numberOfReplicas + 1;
         DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
         for (int i = 0; i < numberOfDataNodes + 1; i++) {
             final DiscoveryNode node = newNode(i);
             discoBuilder = discoBuilder.add(node);
         }
         discoBuilder.localNodeId(newNode(0).getId());
-        discoBuilder.masterNodeId(newNode(numberOfDataNodes + 1).getId()); 
+        discoBuilder.masterNodeId(newNode(numberOfDataNodes + 1).getId());
         ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
         state.nodes(discoBuilder);
         Builder routingTableBuilder = RoutingTable.builder();
@@ -316,7 +327,7 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indice
         state.metaData(metadataBuilder);
         state.routingTable(routingTableBuilder.build());
         return state.build();
-    }    
+    }
 
     /**
      * Creates cluster state with and index that has one shard and as many replicas as numberOfReplicas.
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index 08301e99d6a69..7f1b4adf8df1e 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -163,7 +163,7 @@ public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
+        transportService = transport.createCapturingTransportService(clusterService.getSettings(), threadPool,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
@@ -587,8 +587,6 @@ public void onFailure(Exception e) {
 
     public void testPrimaryReference() throws Exception {
         final IndexShard shard = mock(IndexShard.class);
-        final long primaryTerm = 1 + randomInt(200);
-        when(shard.getPrimaryTerm()).thenReturn(primaryTerm);
 
         AtomicBoolean closed = new AtomicBoolean();
         Releasable releasable = () -> {
@@ -683,9 +681,9 @@ public void testSeqNoIsSetOnPrimary() throws Exception {
 
 
         final IndexShard shard = mock(IndexShard.class);
-        when(shard.getPrimaryTerm()).thenReturn(primaryTerm);
+        when(shard.getPendingPrimaryTerm()).thenReturn(primaryTerm);
         when(shard.routingEntry()).thenReturn(routingEntry);
-        when(shard.isPrimaryMode()).thenReturn(true);
+        when(shard.isRelocatedPrimary()).thenReturn(false);
         IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().shardRoutingTable(shardId);
         Set<String> inSyncIds = randomBoolean() ? Collections.singleton(routingEntry.allocationId().getId()) :
             clusterService.state().metaData().index(index).inSyncAllocationIds(0);
@@ -1201,7 +1199,7 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService
         doAnswer(invocation -> {
             long term = (Long)invocation.getArguments()[0];
             ActionListener<Releasable> callback = (ActionListener<Releasable>) invocation.getArguments()[2];
-            final long primaryTerm = indexShard.getPrimaryTerm();
+            final long primaryTerm = indexShard.getPendingPrimaryTerm();
             if (term < primaryTerm) {
                 throw new IllegalArgumentException(String.format(Locale.ROOT, "%s operation term [%d] is too old (current [%d])",
                     shardId, term, primaryTerm));
@@ -1219,9 +1217,9 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService
             }
             return routing;
         });
-        when(indexShard.isPrimaryMode()).thenAnswer(invocationOnMock -> isRelocated.get() == false);
+        when(indexShard.isRelocatedPrimary()).thenAnswer(invocationOnMock -> isRelocated.get());
         doThrow(new AssertionError("failed shard is not supported")).when(indexShard).failShard(anyString(), any(Exception.class));
-        when(indexShard.getPrimaryTerm()).thenAnswer(i ->
+        when(indexShard.getPendingPrimaryTerm()).thenAnswer(i ->
             clusterService.state().metaData().getIndexSafe(shardId.getIndex()).primaryTerm(shardId.id()));
         return indexShard;
     }
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
index bfcc5938a8690..9a571f1e9d8eb 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
@@ -255,8 +255,8 @@ public void testDocumentFailureInShardOperationOnReplica() throws Exception {
 
     public void testReplicaProxy() throws InterruptedException, ExecutionException {
         CapturingTransport transport = new CapturingTransport();
-        TransportService transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
+        TransportService transportService = transport.createCapturingTransportService(clusterService.getSettings(), threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
         ShardStateAction shardStateAction = new ShardStateAction(Settings.EMPTY, clusterService, transportService, null, null, threadPool);
@@ -454,7 +454,7 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService
         doAnswer(invocation -> {
             long term = (Long)invocation.getArguments()[0];
             ActionListener<Releasable> callback = (ActionListener<Releasable>) invocation.getArguments()[1];
-            final long primaryTerm = indexShard.getPrimaryTerm();
+            final long primaryTerm = indexShard.getPendingPrimaryTerm();
             if (term < primaryTerm) {
                 throw new IllegalArgumentException(String.format(Locale.ROOT, "%s operation term [%d] is too old (current [%d])",
                     shardId, term, primaryTerm));
@@ -472,9 +472,9 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService
             }
             return routing;
         });
-        when(indexShard.isPrimaryMode()).thenAnswer(invocationOnMock -> isRelocated.get() == false);
+        when(indexShard.isRelocatedPrimary()).thenAnswer(invocationOnMock -> isRelocated.get());
         doThrow(new AssertionError("failed shard is not supported")).when(indexShard).failShard(anyString(), any(Exception.class));
-        when(indexShard.getPrimaryTerm()).thenAnswer(i ->
+        when(indexShard.getPendingPrimaryTerm()).thenAnswer(i ->
             clusterService.state().metaData().getIndexSafe(shardId.getIndex()).primaryTerm(shardId.id()));
         return indexShard;
     }
diff --git a/server/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
index 8db45cc5508ef..0505143592497 100644
--- a/server/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
@@ -144,16 +143,15 @@ public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()
-        );
+        transportService = transport.createCapturingTransportService(clusterService.getSettings(), THREAD_POOL,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
         action = new TestTransportInstanceSingleOperationAction(
                 Settings.EMPTY,
                 "indices:admin/test",
                 transportService,
-                new ActionFilters(new HashSet<ActionFilter>()),
+                new ActionFilters(new HashSet<>()),
                 new MyResolver(),
                 Request::new
         );
diff --git a/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java b/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
index 978cd556e132f..1e18135f4eb72 100644
--- a/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
+++ b/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
@@ -131,7 +131,7 @@ public void testExceptionAggregation() {
 
         final NodeValidationException e =
                 expectThrows(NodeValidationException.class,
-                    () -> BootstrapChecks.check(defaultContext, true, checks, "testExceptionAggregation"));
+                    () -> BootstrapChecks.check(defaultContext, true, checks));
         assertThat(e, hasToString(allOf(containsString("bootstrap checks failed"), containsString("first"), containsString("second"))));
         final Throwable[] suppressed = e.getSuppressed();
         assertThat(suppressed.length, equalTo(2));
@@ -162,7 +162,7 @@ long getMaxHeapSize() {
         final NodeValidationException e =
                 expectThrows(
                         NodeValidationException.class,
-                        () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testHeapSizeCheck"));
+                        () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(
                 e.getMessage(),
                 containsString("initial heap size [" + initialHeapSize.get() + "] " +
@@ -170,7 +170,7 @@ long getMaxHeapSize() {
 
         initialHeapSize.set(maxHeapSize.get());
 
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testHeapSizeCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // nothing should happen if the initial heap size or the max
         // heap size is not available
@@ -179,7 +179,7 @@ long getMaxHeapSize() {
         } else {
             maxHeapSize.set(0);
         }
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testHeapSizeCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testFileDescriptorLimits() throws NodeValidationException {
@@ -205,17 +205,17 @@ long getMaxFileDescriptorCount() {
 
         final NodeValidationException e =
                 expectThrows(NodeValidationException.class,
-                        () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testFileDescriptorLimits"));
+                        () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(e.getMessage(), containsString("max file descriptors"));
 
         maxFileDescriptorCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
 
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testFileDescriptorLimits");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // nothing should happen if current file descriptor count is
         // not available
         maxFileDescriptorCount.set(-1);
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testFileDescriptorLimits");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testFileDescriptorLimitsThrowsOnInvalidLimit() {
@@ -262,15 +262,13 @@ boolean isMemoryLocked() {
                         () -> BootstrapChecks.check(
                                 bootstrapContext,
                                 true,
-                                Collections.singletonList(check),
-                                "testFileDescriptorLimitsThrowsOnInvalidLimit"));
+                                Collections.singletonList(check)));
                 assertThat(
                         e.getMessage(),
                         containsString("memory locking requested for elasticsearch process but memory is not locked"));
             } else {
                 // nothing should happen
-                BootstrapChecks.check(bootstrapContext, true, Collections.singletonList(check),
-                    "testFileDescriptorLimitsThrowsOnInvalidLimit");
+                BootstrapChecks.check(bootstrapContext, true, Collections.singletonList(check));
             }
         }
     }
@@ -287,17 +285,17 @@ long getMaxNumberOfThreads() {
 
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck"));
+                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(e.getMessage(), containsString("max number of threads"));
 
         maxNumberOfThreads.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
 
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // nothing should happen if current max number of threads is
         // not available
         maxNumberOfThreads.set(-1);
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testMaxSizeVirtualMemory() throws NodeValidationException {
@@ -317,16 +315,16 @@ long getRlimInfinity() {
 
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxSizeVirtualMemory"));
+                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(e.getMessage(), containsString("max size virtual memory"));
 
         maxSizeVirtualMemory.set(rlimInfinity);
 
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxSizeVirtualMemory");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // nothing should happen if max size virtual memory is not available
         maxSizeVirtualMemory.set(Long.MIN_VALUE);
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxSizeVirtualMemory");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testMaxFileSizeCheck() throws NodeValidationException {
@@ -346,16 +344,16 @@ long getRlimInfinity() {
 
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxFileSize"));
+                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(e.getMessage(), containsString("max file size"));
 
         maxFileSize.set(rlimInfinity);
 
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxFileSize");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // nothing should happen if max file size is not available
         maxFileSize.set(Long.MIN_VALUE);
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxFileSize");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testClientJvmCheck() throws NodeValidationException {
@@ -369,14 +367,14 @@ String getVmName() {
 
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testClientJvmCheck"));
+                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(
                 e.getMessage(),
                 containsString("JVM is using the client VM [Java HotSpot(TM) 32-Bit Client VM] " +
                         "but should be using a server VM for the best performance"));
 
         vmName.set("Java HotSpot(TM) 32-Bit Server VM");
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testClientJvmCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testUseSerialGCCheck() throws NodeValidationException {
@@ -390,14 +388,14 @@ String getUseSerialGC() {
 
         final NodeValidationException e = expectThrows(
             NodeValidationException.class,
-            () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testUseSerialGCCheck"));
+            () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(
             e.getMessage(),
             containsString("JVM is using the serial collector but should not be for the best performance; " + "" +
                 "either it's the default for the VM [" + JvmInfo.jvmInfo().getVmName() +"] or -XX:+UseSerialGC was explicitly specified"));
 
         useSerialGC.set("false");
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testUseSerialGCCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testSystemCallFilterCheck() throws NodeValidationException {
@@ -414,15 +412,14 @@ boolean isSystemCallFilterInstalled() {
 
         final NodeValidationException e = expectThrows(
             NodeValidationException.class,
-            () -> BootstrapChecks.check(context, true, Collections.singletonList(systemCallFilterEnabledCheck),
-                "testSystemCallFilterCheck"));
+            () -> BootstrapChecks.check(context, true, Collections.singletonList(systemCallFilterEnabledCheck)));
         assertThat(
             e.getMessage(),
             containsString("system call filters failed to install; " +
                 "check the logs and fix your configuration or disable system call filters at your own risk"));
 
         isSystemCallFilterInstalled.set(true);
-        BootstrapChecks.check(context, true, Collections.singletonList(systemCallFilterEnabledCheck), "testSystemCallFilterCheck");
+        BootstrapChecks.check(context, true, Collections.singletonList(systemCallFilterEnabledCheck));
         BootstrapContext context_1 = new BootstrapContext(Settings.builder().put("bootstrap.system_call_filter", false).build(), null);
         final BootstrapChecks.SystemCallFilterCheck systemCallFilterNotEnabledCheck = new BootstrapChecks.SystemCallFilterCheck() {
             @Override
@@ -431,9 +428,9 @@ boolean isSystemCallFilterInstalled() {
             }
         };
         isSystemCallFilterInstalled.set(false);
-        BootstrapChecks.check(context_1, true, Collections.singletonList(systemCallFilterNotEnabledCheck), "testSystemCallFilterCheck");
+        BootstrapChecks.check(context_1, true, Collections.singletonList(systemCallFilterNotEnabledCheck));
         isSystemCallFilterInstalled.set(true);
-        BootstrapChecks.check(context_1, true, Collections.singletonList(systemCallFilterNotEnabledCheck), "testSystemCallFilterCheck");
+        BootstrapChecks.check(context_1, true, Collections.singletonList(systemCallFilterNotEnabledCheck));
     }
 
     public void testMightForkCheck() throws NodeValidationException {
@@ -528,8 +525,6 @@ private void runMightForkTest(
         final Runnable enableMightFork,
         final Consumer<NodeValidationException> consumer) throws NodeValidationException {
 
-        final String methodName = Thread.currentThread().getStackTrace()[2].getMethodName();
-
         // if system call filter is disabled, nothing should happen
         isSystemCallFilterInstalled.set(false);
         if (randomBoolean()) {
@@ -537,13 +532,13 @@ private void runMightForkTest(
         } else {
             enableMightFork.run();
         }
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), methodName);
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // if system call filter is enabled, but we will not fork, nothing should
         // happen
         isSystemCallFilterInstalled.set(true);
         disableMightFork.run();
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), methodName);
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // if system call filter is enabled, and we might fork, the check should be enforced, regardless of bootstrap checks being enabled
         // or not
@@ -552,7 +547,7 @@ private void runMightForkTest(
 
         final NodeValidationException e = expectThrows(
             NodeValidationException.class,
-            () -> BootstrapChecks.check(defaultContext, randomBoolean(), Collections.singletonList(check), methodName));
+            () -> BootstrapChecks.check(defaultContext, randomBoolean(), Collections.singletonList(check)));
         consumer.accept(e);
     }
 
@@ -577,7 +572,7 @@ String javaVersion() {
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
                 () -> {
-                    BootstrapChecks.check(defaultContext, true, checks, "testEarlyAccessCheck");
+                    BootstrapChecks.check(defaultContext, true, checks);
                 });
         assertThat(
                 e.getMessage(),
@@ -588,7 +583,7 @@ String javaVersion() {
 
         // if not on an early-access build, nothing should happen
         javaVersion.set(randomFrom("1.8.0_152", "9"));
-        BootstrapChecks.check(defaultContext, true, checks, "testEarlyAccessCheck");
+        BootstrapChecks.check(defaultContext, true, checks);
 
     }
 
@@ -624,7 +619,7 @@ boolean isJava8() {
         final NodeValidationException e =
             expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck), "testG1GCCheck"));
+                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck)));
         assertThat(
             e.getMessage(),
             containsString(
@@ -632,12 +627,12 @@ boolean isJava8() {
 
         // if G1GC is disabled, nothing should happen
         isG1GCEnabled.set(false);
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck), "testG1GCCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck));
 
         // if on or after update 40, nothing should happen independent of whether or not G1GC is enabled
         isG1GCEnabled.set(randomBoolean());
         jvmVersion.set(String.format(Locale.ROOT, "25.%d-b%d", randomIntBetween(40, 112), randomIntBetween(1, 128)));
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck), "testG1GCCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck));
 
         final BootstrapChecks.G1GCCheck nonOracleCheck = new BootstrapChecks.G1GCCheck() {
 
@@ -649,7 +644,7 @@ String jvmVendor() {
         };
 
         // if not on an Oracle JVM, nothing should happen
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(nonOracleCheck), "testG1GCCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(nonOracleCheck));
 
         final BootstrapChecks.G1GCCheck nonJava8Check = new BootstrapChecks.G1GCCheck() {
 
@@ -661,7 +656,7 @@ boolean isJava8() {
         };
 
         // if not Java 8, nothing should happen
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(nonJava8Check), "testG1GCCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(nonJava8Check));
     }
 
     public void testAllPermissionCheck() throws NodeValidationException {
@@ -676,12 +671,12 @@ boolean isAllPermissionGranted() {
         final List<BootstrapCheck> checks = Collections.singletonList(allPermissionCheck);
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, checks, "testIsAllPermissionCheck"));
+                () -> BootstrapChecks.check(defaultContext, true, checks));
         assertThat(e, hasToString(containsString("granting the all permission effectively disables security")));
 
         // if all permissions are not granted, nothing should happen
         isAllPermissionGranted.set(false);
-        BootstrapChecks.check(defaultContext, true, checks, "testIsAllPermissionCheck");
+        BootstrapChecks.check(defaultContext, true, checks);
     }
 
     public void testAlwaysEnforcedChecks() {
@@ -699,7 +694,7 @@ public boolean alwaysEnforce() {
 
         final NodeValidationException alwaysEnforced = expectThrows(
             NodeValidationException.class,
-            () -> BootstrapChecks.check(defaultContext, randomBoolean(), Collections.singletonList(check), "testAlwaysEnforcedChecks"));
+            () -> BootstrapChecks.check(defaultContext, randomBoolean(), Collections.singletonList(check)));
         assertThat(alwaysEnforced, hasToString(containsString("error")));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java b/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
index e2bf07b7d0bb4..a6e50170d7e80 100644
--- a/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
+++ b/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
@@ -65,7 +65,7 @@ public void testUncaughtError() throws InterruptedException {
         final AtomicInteger observedStatus = new AtomicInteger();
         final AtomicReference<String> threadNameReference = new AtomicReference<>();
         final AtomicReference<Throwable> throwableReference = new AtomicReference<>();
-        thread.setUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler(() -> "testUncaughtError") {
+        thread.setUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler() {
 
             @Override
             void halt(int status) {
@@ -106,7 +106,7 @@ public void testUncaughtException() throws InterruptedException {
         thread.setName(name);
         final AtomicReference<String> threadNameReference = new AtomicReference<>();
         final AtomicReference<Throwable> throwableReference = new AtomicReference<>();
-        thread.setUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler(() -> "testUncaughtException") {
+        thread.setUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler() {
             @Override
             void halt(int status) {
                 fail();
diff --git a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
index 669a678b77e31..513f07b733cda 100644
--- a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
+++ b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.client.transport;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse;
 import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
@@ -26,13 +27,13 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.transport.CloseableConnection;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.RequestHandlerRegistry;
@@ -43,9 +44,9 @@
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseHandler;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportStats;
 
-import java.io.IOException;
 import java.net.UnknownHostException;
 import java.util.Collections;
 import java.util.Map;
@@ -59,7 +60,7 @@ abstract class FailAndRetryMockTransport<Response extends TransportResponse> imp
     private final Random random;
     private final ClusterName clusterName;
     private volatile Map<String, RequestHandlerRegistry> requestHandlers = Collections.emptyMap();
-    final Object requestHandlerMutex = new Object();
+    private final Object requestHandlerMutex = new Object();
     private final ResponseHandlers responseHandlers = new ResponseHandlers();
     private TransportConnectionListener listener;
 
@@ -78,8 +79,9 @@ abstract class FailAndRetryMockTransport<Response extends TransportResponse> imp
     protected abstract ClusterState getMockClusterState(DiscoveryNode node);
 
     @Override
-    public Connection getConnection(DiscoveryNode node) {
-        return new Connection() {
+    public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) {
+        return new CloseableConnection() {
+
             @Override
             public DiscoveryNode getNode() {
                 return node;
@@ -87,19 +89,22 @@ public DiscoveryNode getNode() {
 
             @Override
             public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
-                throws IOException, TransportException {
-
+                throws TransportException {
                 //we make sure that nodes get added to the connected ones when calling addTransportAddress, by returning proper nodes info
                 if (connectMode) {
                     if (TransportLivenessAction.NAME.equals(action)) {
                         TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener);
-                        transportResponseHandler.handleResponse(new LivenessResponse(ClusterName.CLUSTER_NAME_SETTING.
-                            getDefault(Settings.EMPTY),
-                            node));
+                        ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY);
+                        transportResponseHandler.handleResponse(new LivenessResponse(clusterName, node));
                     } else if (ClusterStateAction.NAME.equals(action)) {
                         TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener);
                         ClusterState clusterState = getMockClusterState(node);
                         transportResponseHandler.handleResponse(new ClusterStateResponse(clusterName, clusterState, 0L));
+                    } else if (TransportService.HANDSHAKE_ACTION_NAME.equals(action)) {
+                        TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener);
+                        Version version = node.getVersion();
+                        transportResponseHandler.handleResponse(new TransportService.HandshakeResponse(node, clusterName, version));
+
                     } else {
                         throw new UnsupportedOperationException("Mock transport does not understand action " + action);
                     }
@@ -129,19 +134,9 @@ public void sendRequest(long requestId, String action, TransportRequest request,
                     }
                 }
             }
-
-            @Override
-            public void close() throws IOException {
-
-            }
         };
     }
 
-    @Override
-    public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
-        return getConnection(node);
-    }
-
     protected abstract Response newResponse();
 
     public void endConnectMode() {
@@ -175,23 +170,6 @@ public TransportAddress[] addressesFromString(String address, int perAddressLimi
         throw new UnsupportedOperationException();
     }
 
-    @Override
-    public boolean nodeConnected(DiscoveryNode node) {
-        return false;
-    }
-
-    @Override
-    public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
-                              CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
-        throws ConnectTransportException {
-
-    }
-
-    @Override
-    public void disconnectFromNode(DiscoveryNode node) {
-
-    }
-
     @Override
     public Lifecycle.State lifecycleState() {
         return null;
diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
index 2beaed1e106e4..afc6b47483eba 100644
--- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
@@ -41,7 +41,6 @@
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
@@ -51,7 +50,6 @@
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseHandler;
-import org.elasticsearch.transport.TransportService;
 import org.hamcrest.CustomMatcher;
 
 import java.io.Closeable;
@@ -83,7 +81,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
     private static class TestIteration implements Closeable {
         private final ThreadPool threadPool;
         private final FailAndRetryMockTransport<TestResponse> transport;
-        private final TransportService transportService;
+        private final MockTransportService transportService;
         private final TransportClientNodesService transportClientNodesService;
         private final int listNodesCount;
         private final int sniffNodesCount;
@@ -143,7 +141,8 @@ protected ClusterState getMockClusterState(DiscoveryNode node) {
                     return ClusterState.builder(clusterName).nodes(TestIteration.this.nodeMap.get(node.getAddress())).build();
                 }
             };
-            transportService = new TransportService(settings, transport, threadPool, new TransportInterceptor() {
+
+            transportService = new MockTransportService(settings, transport, threadPool, new TransportInterceptor() {
                 @Override
                 public AsyncSender interceptSender(AsyncSender sender) {
                     return new AsyncSender() {
@@ -165,6 +164,11 @@ public <T extends TransportResponse> void sendRequest(Transport.Connection conne
                 assert addr == null : "boundAddress: " + addr;
                 return DiscoveryNode.createLocal(settings, buildNewFakeTransportAddress(), UUIDs.randomBase64UUID());
             }, null, Collections.emptySet());
+            transportService.addNodeConnectedBehavior((connectionManager, discoveryNode) -> false);
+            transportService.addGetConnectionBehavior((connectionManager, discoveryNode) -> {
+                // The FailAndRetryTransport does not use the connection profile
+                return transport.openConnection(discoveryNode, null);
+            });
             transportService.start();
             transportService.acceptIncomingRequests();
             transportClientNodesService =
@@ -356,25 +360,15 @@ public void testSniffNodesSamplerClosesConnections() throws Exception {
                     .build();
 
             try (MockTransportService clientService = createNewService(clientSettings, Version.CURRENT, threadPool, null)) {
-                final List<MockConnection> establishedConnections = new CopyOnWriteArrayList<>();
-                final List<MockConnection> reusedConnections = new CopyOnWriteArrayList<>();
-
-                clientService.addDelegate(remoteService,  new MockTransportService.DelegateTransport(clientService.original()) {
-                    @Override
-                    public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
-                        MockConnection connection = new MockConnection(super.openConnection(node, profile));
-                        establishedConnections.add(connection);
-                        return connection;
-                    }
+                final List<Transport.Connection> establishedConnections = new CopyOnWriteArrayList<>();
 
-                    @Override
-                    public Connection getConnection(DiscoveryNode node) {
-                        MockConnection connection = new MockConnection(super.getConnection(node));
-                        reusedConnections.add(connection);
-                        return connection;
-                    }
+                clientService.addConnectBehavior(remoteService, (transport, discoveryNode, profile) -> {
+                    Transport.Connection connection = transport.openConnection(discoveryNode, profile);
+                    establishedConnections.add(connection);
+                    return connection;
                 });
 
+
                 clientService.start();
                 clientService.acceptIncomingRequests();
 
@@ -382,27 +376,26 @@ public Connection getConnection(DiscoveryNode node) {
                         new TransportClientNodesService(clientSettings, clientService, threadPool, (a, b) -> {})) {
                     assertEquals(0, transportClientNodesService.connectedNodes().size());
                     assertEquals(0, establishedConnections.size());
-                    assertEquals(0, reusedConnections.size());
 
                     transportClientNodesService.addTransportAddresses(remoteService.getLocalDiscoNode().getAddress());
                     assertEquals(1, transportClientNodesService.connectedNodes().size());
-                    assertClosedConnections(establishedConnections, 1);
+                    assertEquals(1, clientService.connectionManager().connectedNodeCount());
 
                     transportClientNodesService.doSample();
-                    assertClosedConnections(establishedConnections, 2);
-                    assertOpenConnections(reusedConnections, 1);
+                    assertEquals(1, clientService.connectionManager().connectedNodeCount());
 
+                    establishedConnections.clear();
                     handler.blockRequest();
                     Thread thread = new Thread(transportClientNodesService::doSample);
                     thread.start();
 
-                    assertBusy(() ->  assertEquals(3, establishedConnections.size()));
-                    assertFalse("Temporary ping connection must be opened", establishedConnections.get(2).isClosed());
+                    assertBusy(() ->  assertTrue(establishedConnections.size() >= 1));
+                    assertFalse("Temporary ping connection must be opened", establishedConnections.get(0).isClosed());
 
                     handler.releaseRequest();
                     thread.join();
 
-                    assertClosedConnections(establishedConnections, 3);
+                    assertTrue(establishedConnections.get(0).isClosed());
                 }
             }
         } finally {
@@ -410,56 +403,6 @@ public Connection getConnection(DiscoveryNode node) {
         }
     }
 
-    private void assertClosedConnections(final List<MockConnection> connections, final int size) {
-        assertEquals("Expecting " + size + " closed connections but got " + connections.size(), size, connections.size());
-        connections.forEach(c -> assertConnection(c, true));
-    }
-
-    private void assertOpenConnections(final List<MockConnection> connections, final int size) {
-        assertEquals("Expecting " + size + " open connections but got " + connections.size(), size, connections.size());
-        connections.forEach(c -> assertConnection(c, false));
-    }
-
-    private static void assertConnection(final MockConnection connection, final boolean closed) {
-        assertEquals("Connection [" + connection + "] must be " + (closed ? "closed" : "open"), closed, connection.isClosed());
-    }
-
-    class MockConnection implements Transport.Connection {
-        private final AtomicBoolean closed = new AtomicBoolean(false);
-        private final Transport.Connection connection;
-
-        private MockConnection(Transport.Connection connection) {
-            this.connection = connection;
-        }
-
-        @Override
-        public DiscoveryNode getNode() {
-            return connection.getNode();
-        }
-
-        @Override
-        public Version getVersion() {
-            return connection.getVersion();
-        }
-
-        @Override
-        public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
-                throws IOException, TransportException {
-            connection.sendRequest(requestId, action, request, options);
-        }
-
-        @Override
-        public void close() throws IOException {
-            if (closed.compareAndSet(false, true)) {
-                connection.close();
-            }
-        }
-
-        boolean isClosed() {
-            return closed.get();
-        }
-    }
-
     class MockHandler implements TransportRequestHandler<ClusterStateRequest> {
         private final AtomicBoolean block = new AtomicBoolean(false);
         private final CountDownLatch release = new CountDownLatch(1);
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
index d7d232a08d43c..b17a0cc5418e9 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
@@ -46,12 +46,9 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 import org.hamcrest.Matchers;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
@@ -116,23 +113,23 @@ public void blockActions(String... actions) {
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
-                // manual collection or upon cluster forming.
-                .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2)
-                .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), "1s")
-                .build();
+            // manual collection or upon cluster forming.
+            .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2)
+            .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), "1s")
+            .build();
     }
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(TestPlugin.class,
-                MockTransportService.TestPlugin.class);
+            MockTransportService.TestPlugin.class);
     }
 
     public void testClusterInfoServiceCollectsInformation() throws Exception {
         internalCluster().startNodes(2);
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
-                .put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0)
-                .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE).build()));
+            .put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0)
+            .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE).build()));
         ensureGreen("test");
         InternalTestCluster internalTestCluster = internalCluster();
         // Get the cluster info service on the master node
@@ -177,8 +174,8 @@ public void testClusterInfoServiceCollectsInformation() throws Exception {
 
     public void testClusterInfoServiceInformationClearOnError() throws InterruptedException, ExecutionException {
         internalCluster().startNodes(2,
-                // manually control publishing
-                Settings.builder().put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.getKey(), "60m").build());
+            // manually control publishing
+            Settings.builder().put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.getKey(), "60m").build());
         prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)).get();
         ensureGreen("test");
         InternalTestCluster internalTestCluster = internalCluster();
@@ -196,19 +193,16 @@ public void testClusterInfoServiceInformationClearOnError() throws InterruptedEx
         final Set<String> blockedActions = newHashSet(NodesStatsAction.NAME, NodesStatsAction.NAME + "[n]", IndicesStatsAction.NAME, IndicesStatsAction.NAME + "[n]");
         // drop all outgoing stats requests to force a timeout.
         for (DiscoveryNode node : internalTestCluster.clusterService().state().getNodes()) {
-            mockTransportService.addDelegate(internalTestCluster.getInstance(TransportService.class, node.getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {
-                @Override
-                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
-                                           TransportRequestOptions options) throws IOException {
+            mockTransportService.addSendBehavior(internalTestCluster.getInstance(TransportService.class, node.getName()),
+                (connection, requestId, action, request, options) -> {
                     if (blockedActions.contains(action)) {
                         if (timeout.get()) {
                             logger.info("dropping [{}] to [{}]", action, node);
                             return;
                         }
                     }
-                    super.sendRequest(connection, requestId, action, request, options);
-                }
-            });
+                    connection.sendRequest(requestId, action, request, options);
+                });
         }
 
         // timeouts shouldn't clear the info
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
index 4c7a42de2ee95..af3807226a943 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
@@ -63,7 +63,7 @@
 public class ClusterModuleTests extends ModuleTestCase {
     private ClusterInfoService clusterInfoService = EmptyClusterInfoService.INSTANCE;
     private ClusterService clusterService = new ClusterService(Settings.EMPTY,
-        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, Collections.emptyMap());
+        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null);
     static class FakeAllocationDecider extends AllocationDecider {
         protected FakeAllocationDecider(Settings settings) {
             super(settings);
@@ -202,57 +202,6 @@ public void testAllocationDeciderOrder() {
         }
     }
 
-    public void testCustomSuppliers() {
-        Map<String, Supplier<ClusterState.Custom>> customSuppliers = ClusterModule.getClusterStateCustomSuppliers(Collections.emptyList());
-        assertEquals(3, customSuppliers.size());
-        assertTrue(customSuppliers.containsKey(SnapshotsInProgress.TYPE));
-        assertTrue(customSuppliers.containsKey(SnapshotDeletionsInProgress.TYPE));
-        assertTrue(customSuppliers.containsKey(RestoreInProgress.TYPE));
-
-        customSuppliers = ClusterModule.getClusterStateCustomSuppliers(Collections.singletonList(new ClusterPlugin() {
-            @Override
-            public Map<String, Supplier<ClusterState.Custom>> getInitialClusterStateCustomSupplier() {
-                return Collections.singletonMap("foo", () -> null);
-            }
-        }));
-        assertEquals(4, customSuppliers.size());
-        assertTrue(customSuppliers.containsKey(SnapshotsInProgress.TYPE));
-        assertTrue(customSuppliers.containsKey(SnapshotDeletionsInProgress.TYPE));
-        assertTrue(customSuppliers.containsKey(RestoreInProgress.TYPE));
-        assertTrue(customSuppliers.containsKey("foo"));
-
-        {
-            // Eclipse Neon 2 didn't compile the plugins definition inside the lambda expression,
-            // probably due to https://bugs.eclipse.org/bugs/show_bug.cgi?id=511750, which is
-            // fixed in Eclipse Oxygon. Pulled out the plugins definition to make it work in older versions
-            List<ClusterPlugin> plugins = Collections.singletonList(new ClusterPlugin() {
-                @Override
-                public Map<String, Supplier<ClusterState.Custom>> getInitialClusterStateCustomSupplier() {
-                    return Collections.singletonMap(SnapshotsInProgress.TYPE, () -> null);
-                }
-            });
-            IllegalStateException ise = expectThrows(IllegalStateException.class,
-                    () -> ClusterModule.getClusterStateCustomSuppliers(plugins));
-            assertEquals(ise.getMessage(), "custom supplier key [snapshots] is registered more than once");
-        }
-        {
-            List<ClusterPlugin> plugins = Arrays.asList(new ClusterPlugin() {
-                @Override
-                public Map<String, Supplier<ClusterState.Custom>> getInitialClusterStateCustomSupplier() {
-                    return Collections.singletonMap("foo", () -> null);
-                }
-            }, new ClusterPlugin() {
-                @Override
-                public Map<String, Supplier<ClusterState.Custom>> getInitialClusterStateCustomSupplier() {
-                    return Collections.singletonMap("foo", () -> null);
-                }
-            });
-            IllegalStateException ise = expectThrows(IllegalStateException.class,
-                    () -> ClusterModule.getClusterStateCustomSuppliers(plugins));
-            assertEquals(ise.getMessage(), "custom supplier key [foo] is registered more than once");
-        }
-    }
-
     public void testPre63CustomsFiltering() {
         final String whiteListedClusterCustom = randomFrom(ClusterModule.PRE_6_3_CLUSTER_CUSTOMS_WHITE_LIST);
         final String whiteListedMetaDataCustom = randomFrom(ClusterModule.PRE_6_3_METADATA_CUSTOMS_WHITE_LIST);
diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index eb93dad5db8a0..92c1016d3e615 100644
--- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -20,16 +20,16 @@
 package org.elasticsearch.cluster;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -39,6 +39,7 @@
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportConnectionListener;
 import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -46,8 +47,6 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.io.IOException;
-import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -56,6 +55,8 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.Predicate;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -70,7 +71,7 @@ private List<DiscoveryNode> generateNodes() {
         for (int i = randomIntBetween(20, 50); i > 0; i--) {
             Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())));
             nodes.add(new DiscoveryNode("node_" + i, "" + i, buildNewFakeTransportAddress(), Collections.emptyMap(),
-                    roles, Version.CURRENT));
+                roles, Version.CURRENT));
         }
         return nodes;
     }
@@ -121,7 +122,7 @@ public void testReconnect() {
         for (int i = 0; i < 3; i++) {
             // simulate disconnects
             for (DiscoveryNode node : randomSubsetOf(nodes)) {
-                transport.disconnectFromNode(node);
+                transportService.disconnectFromNode(node);
             }
             service.new ConnectionChecker().run();
         }
@@ -134,18 +135,12 @@ public void testReconnect() {
 
     private void assertConnectedExactlyToNodes(ClusterState state) {
         assertConnected(state.nodes());
-        assertThat(transport.connectedNodes.size(), equalTo(state.nodes().getSize()));
+        assertThat(transportService.getConnectionManager().connectedNodeCount(), equalTo(state.nodes().getSize()));
     }
 
     private void assertConnected(Iterable<DiscoveryNode> nodes) {
         for (DiscoveryNode node : nodes) {
-            assertTrue("not connected to " + node, transport.connectedNodes.contains(node));
-        }
-    }
-
-    private void assertNotConnected(Iterable<DiscoveryNode> nodes) {
-        for (DiscoveryNode node : nodes) {
-            assertFalse("still connected to " + node, transport.connectedNodes.contains(node));
+            assertTrue("not connected to " + node, transportService.nodeConnected(node));
         }
     }
 
@@ -155,7 +150,8 @@ public void setUp() throws Exception {
         super.setUp();
         this.threadPool = new TestThreadPool(getClass().getName());
         this.transport = new MockTransport();
-        transportService = new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+        transportService = new NoHandshakeTransportService(Settings.EMPTY, transport, threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR,
             boundAddress -> DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), UUIDs.randomBase64UUID()), null,
             Collections.emptySet());
         transportService.start();
@@ -171,11 +167,29 @@ public void tearDown() throws Exception {
         super.tearDown();
     }
 
-    final class MockTransport implements Transport {
-        Set<DiscoveryNode> connectedNodes = ConcurrentCollections.newConcurrentSet();
-        volatile boolean randomConnectionExceptions = false;
+    private final class NoHandshakeTransportService extends TransportService {
+
+        private NoHandshakeTransportService(Settings settings,
+                                            Transport transport,
+                                            ThreadPool threadPool,
+                                            TransportInterceptor transportInterceptor,
+                                            Function<BoundTransportAddress, DiscoveryNode> localNodeFactory,
+                                            ClusterSettings clusterSettings,
+                                            Set<String> taskHeaders) {
+            super(settings, transport, threadPool, transportInterceptor, localNodeFactory, clusterSettings, taskHeaders);
+        }
+
+        @Override
+        public HandshakeResponse handshake(Transport.Connection connection, long timeout, Predicate<ClusterName> clusterNamePredicate) {
+            return new HandshakeResponse(connection.getNode(), new ClusterName(""), Version.CURRENT);
+        }
+    }
+
+    private final class MockTransport implements Transport {
         private ResponseHandlers responseHandlers = new ResponseHandlers();
-        private TransportConnectionListener listener = new TransportConnectionListener() {};
+        private volatile boolean randomConnectionExceptions = false;
+        private TransportConnectionListener listener = new TransportConnectionListener() {
+        };
 
         @Override
         public <Request extends TransportRequest> void registerRequestHandler(RequestHandlerRegistry<Request> reg) {
@@ -207,37 +221,19 @@ public Map<String, BoundTransportAddress> profileBoundAddresses() {
         }
 
         @Override
-        public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
+        public TransportAddress[] addressesFromString(String address, int perAddressLimit) {
             return new TransportAddress[0];
         }
 
         @Override
-        public boolean nodeConnected(DiscoveryNode node) {
-            return connectedNodes.contains(node);
-        }
-
-        @Override
-        public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
-                                  CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
-            throws ConnectTransportException {
+        public Connection openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) {
             if (connectionProfile == null) {
-                if (connectedNodes.contains(node) == false && randomConnectionExceptions && randomBoolean()) {
+                if (randomConnectionExceptions && randomBoolean()) {
                     throw new ConnectTransportException(node, "simulated");
                 }
-                connectedNodes.add(node);
                 listener.onNodeConnected(node);
             }
-        }
-
-        @Override
-        public void disconnectFromNode(DiscoveryNode node) {
-            connectedNodes.remove(node);
-            listener.onNodeDisconnected(node);
-        }
-
-        @Override
-        public Connection getConnection(DiscoveryNode node) {
-            return new Connection() {
+            Connection connection = new Connection() {
                 @Override
                 public DiscoveryNode getNode() {
                     return node;
@@ -249,16 +245,21 @@ public void sendRequest(long requestId, String action, TransportRequest request,
 
                 }
 
+                @Override
+                public void addCloseListener(ActionListener<Void> listener) {
+
+                }
+
                 @Override
                 public void close() {
 
                 }
-            };
-        }
 
-        @Override
-        public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) {
-            Connection connection = getConnection(node);
+                @Override
+                public boolean isClosed() {
+                    return false;
+                }
+            };
             listener.onConnectionOpened(connection);
             return connection;
         }
@@ -282,13 +283,16 @@ public void removeLifecycleListener(LifecycleListener listener) {
         }
 
         @Override
-        public void start() {}
+        public void start() {
+        }
 
         @Override
-        public void stop() {}
+        public void stop() {
+        }
 
         @Override
-        public void close() {}
+        public void close() {
+        }
 
         @Override
         public TransportStats getStats() {
diff --git a/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
index 0ba3de4381891..606e716d210b2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -38,14 +39,20 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.plugins.ClusterPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.CollectionAssertions;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -53,9 +60,9 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
-import java.util.function.Supplier;
+import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertIndexTemplateExists;
 import static org.hamcrest.Matchers.equalTo;
@@ -279,13 +286,11 @@ public void testIndicesIgnoreUnavailableFalse() throws Exception {
         }
     }
 
-    public void testPrivateCustomsAreExcluded() {
+    public void testPrivateCustomsAreExcluded() throws Exception {
+        // ensure that the custom is injected into the cluster state
+        assertBusy(() -> assertTrue(clusterService().state().customs().containsKey("test")));
         ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().setCustoms(true).get();
         assertFalse(clusterStateResponse.getState().customs().containsKey("test"));
-        // just to make sure there is something
-        assertTrue(clusterStateResponse.getState().customs().containsKey(SnapshotDeletionsInProgress.TYPE));
-        ClusterState state = internalCluster().getInstance(ClusterService.class).state();
-        assertTrue(state.customs().containsKey("test"));
     }
 
     private static class TestCustom extends AbstractNamedDiffable<ClusterState.Custom> implements ClusterState.Custom {
@@ -333,11 +338,6 @@ public static class PrivateCustomPlugin extends Plugin implements ClusterPlugin
 
         public PrivateCustomPlugin() {}
 
-        @Override
-        public Map<String, Supplier<ClusterState.Custom>> getInitialClusterStateCustomSupplier() {
-            return Collections.singletonMap("test", () -> new TestCustom(1));
-        }
-
         @Override
         public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
@@ -345,5 +345,54 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             entries.add(new NamedWriteableRegistry.Entry(NamedDiff.class, "test", TestCustom::readDiffFrom));
             return entries;
         }
+
+        private final AtomicBoolean installed = new AtomicBoolean();
+
+        @Override
+        public Collection<Object> createComponents(
+            final Client client,
+            final ClusterService clusterService,
+            final ThreadPool threadPool,
+            final ResourceWatcherService resourceWatcherService,
+            final ScriptService scriptService,
+            final NamedXContentRegistry xContentRegistry,
+            final Environment environment,
+            final NodeEnvironment nodeEnvironment,
+            final NamedWriteableRegistry namedWriteableRegistry) {
+            clusterService.addListener(event -> {
+                final ClusterState state = event.state();
+                if (state.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) {
+                    return;
+                }
+
+                if (state.nodes().isLocalNodeElectedMaster()) {
+                    if (state.custom("test") == null) {
+                        if (installed.compareAndSet(false, true)) {
+                            clusterService.submitStateUpdateTask("install-metadata-custom", new ClusterStateUpdateTask(Priority.URGENT) {
+
+                                @Override
+                                public ClusterState execute(ClusterState currentState) {
+                                    if (currentState.custom("test") == null) {
+                                        final ClusterState.Builder builder = ClusterState.builder(currentState);
+                                        builder.putCustom("test", new TestCustom(42));
+                                        return builder.build();
+                                    } else {
+                                        return currentState;
+                                    }
+                                }
+
+                                @Override
+                                public void onFailure(String source, Exception e) {
+                                    throw new AssertionError(e);
+                                }
+
+                            });
+                        }
+                    }
+                }
+
+            });
+            return Collections.emptyList();
+        }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index 1d78cdeb98374..64fa51d159a54 100644
--- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -124,8 +124,8 @@ public void setUp() throws Exception {
         super.setUp();
         this.transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
+        transportService = transport.createCapturingTransportService(clusterService.getSettings(), THREAD_POOL,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
         shardStateAction = new TestShardStateAction(Settings.EMPTY, clusterService, transportService, null, null);
diff --git a/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index 0f5f4870ae1bb..32a8c06bb7020 100644
--- a/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -94,7 +94,8 @@ public static void setupThreadPool() {
     public void setUp() throws Exception {
         super.setUp();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(clusterService.getSettings(), new CapturingTransport(), threadPool,
+        CapturingTransport transport = new CapturingTransport();
+        transportService = transport.createCapturingTransportService(clusterService.getSettings(), threadPool,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java
index aa77d7b4bf92f..530e3dd318526 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java
@@ -76,7 +76,7 @@ protected void assertPrimaryTerms(long shard0Term, long shard1Term) {
             if (indexService != null) {
                 for (IndexShard shard : indexService) {
                     assertThat("term mismatch for shard " + shard.shardId(),
-                        shard.getPrimaryTerm(), equalTo(metaData.primaryTerm(shard.shardId().id())));
+                        shard.getPendingPrimaryTerm(), equalTo(metaData.primaryTerm(shard.shardId().id())));
                 }
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java
index f4c2090895b83..0ca47377bdd0d 100644
--- a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java
@@ -118,13 +118,13 @@ public void testClusterStateUpdateLogging() throws Exception {
         mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                         "test1",
-                        clusterApplierService.getClass().getName(),
+                        clusterApplierService.getClass().getCanonicalName(),
                         Level.DEBUG,
                         "*processing [test1]: took [1s] no change in cluster state"));
         mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                         "test2",
-                        clusterApplierService.getClass().getName(),
+                        clusterApplierService.getClass().getCanonicalName(),
                         Level.TRACE,
                         "*failed to execute cluster state applier in [2s]*"));
 
@@ -192,19 +192,19 @@ public void testLongClusterStateUpdateLogging() throws Exception {
         mockAppender.addExpectation(
                 new MockLogAppender.UnseenEventExpectation(
                         "test1 shouldn't see because setting is too low",
-                        clusterApplierService.getClass().getName(),
+                        clusterApplierService.getClass().getCanonicalName(),
                         Level.WARN,
                         "*cluster state applier task [test1] took [*] above the warn threshold of *"));
         mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                         "test2",
-                        clusterApplierService.getClass().getName(),
+                        clusterApplierService.getClass().getCanonicalName(),
                         Level.WARN,
                         "*cluster state applier task [test2] took [32s] above the warn threshold of *"));
         mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                         "test4",
-                        clusterApplierService.getClass().getName(),
+                        clusterApplierService.getClass().getCanonicalName(),
                         Level.WARN,
                         "*cluster state applier task [test3] took [34s] above the warn threshold of *"));
 
@@ -412,7 +412,7 @@ static class TimedClusterApplierService extends ClusterApplierService {
         public volatile Long currentTimeOverride = null;
 
         TimedClusterApplierService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
-            super(settings, clusterSettings, threadPool, () -> ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings)));
+            super(settings, clusterSettings, threadPool);
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/cluster/service/ClusterSerivceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/ClusterSerivceTests.java
deleted file mode 100644
index 2cebd41a52c43..0000000000000
--- a/server/src/test/java/org/elasticsearch/cluster/service/ClusterSerivceTests.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.cluster.service;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.Diff;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-import java.util.Collections;
-
-public class ClusterSerivceTests extends ESTestCase {
-
-    public void testNewBuilderContainsCustoms() {
-        ClusterState.Custom custom = new ClusterState.Custom() {
-            @Override
-            public Diff<ClusterState.Custom> diff(ClusterState.Custom previousState) {
-                return null;
-            }
-
-            @Override
-            public String getWriteableName() {
-                return null;
-            }
-
-            @Override
-            public Version getMinimalSupportedVersion() {
-                return Version.CURRENT;
-            }
-
-            @Override
-            public void writeTo(StreamOutput out) throws IOException {
-
-            }
-
-            @Override
-            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-                return null;
-            }
-        };
-        ClusterService service = new ClusterService(Settings.EMPTY,
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, Collections.singletonMap("foo", () ->
-            custom));
-        ClusterState.Builder builder = service.newClusterStateBuilder();
-        assertSame(builder.build().custom("foo"), custom);
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java
index 1ef548bd68114..1168e1034fe6c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java
@@ -309,19 +309,19 @@ public void testClusterStateUpdateLogging() throws Exception {
         mockAppender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 "test1",
-                masterService.getClass().getName(),
+                masterService.getClass().getCanonicalName(),
                 Level.DEBUG,
                 "*processing [test1]: took [1s] no change in cluster state"));
         mockAppender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 "test2",
-                masterService.getClass().getName(),
+                masterService.getClass().getCanonicalName(),
                 Level.TRACE,
                 "*failed to execute cluster state update in [2s]*"));
         mockAppender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 "test3",
-                masterService.getClass().getName(),
+                masterService.getClass().getCanonicalName(),
                 Level.DEBUG,
                 "*processing [test3]: took [3s] done publishing updated cluster state (version: *, uuid: *)"));
 
@@ -650,25 +650,25 @@ public void testLongClusterStateUpdateLogging() throws Exception {
         mockAppender.addExpectation(
             new MockLogAppender.UnseenEventExpectation(
                 "test1 shouldn't see because setting is too low",
-                masterService.getClass().getName(),
+                masterService.getClass().getCanonicalName(),
                 Level.WARN,
                 "*cluster state update task [test1] took [*] above the warn threshold of *"));
         mockAppender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 "test2",
-                masterService.getClass().getName(),
+                masterService.getClass().getCanonicalName(),
                 Level.WARN,
                 "*cluster state update task [test2] took [32s] above the warn threshold of *"));
         mockAppender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 "test3",
-                masterService.getClass().getName(),
+                masterService.getClass().getCanonicalName(),
                 Level.WARN,
                 "*cluster state update task [test3] took [33s] above the warn threshold of *"));
         mockAppender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 "test4",
-                masterService.getClass().getName(),
+                masterService.getClass().getCanonicalName(),
                 Level.WARN,
                 "*cluster state update task [test4] took [34s] above the warn threshold of *"));
 
diff --git a/server/src/test/java/org/elasticsearch/common/RoundingTests.java b/server/src/test/java/org/elasticsearch/common/RoundingTests.java
new file mode 100644
index 0000000000000..1664f67a44df9
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/RoundingTests.java
@@ -0,0 +1,757 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.rounding.DateTimeUnit;
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+import org.hamcrest.TypeSafeMatcher;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.TemporalAccessor;
+import java.time.zone.ZoneOffsetTransition;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class RoundingTests extends ESTestCase {
+
+    public void testUTCTimeUnitRounding() {
+        Rounding tzRounding = Rounding.builder(Rounding.DateTimeUnit.MONTH_OF_YEAR).build();
+        ZoneId tz = ZoneOffset.UTC;
+        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-01T00:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-01T00:00:00.000Z")), isDate(time("2009-03-01T00:00:00.000Z"), tz));
+
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR).build();
+        assertThat(tzRounding.round(time("2012-01-10T01:01:01")), isDate(time("2012-01-09T00:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2012-01-09T00:00:00.000Z")), isDate(time("2012-01-16T00:00:00.000Z"), tz));
+    }
+
+    public void testUTCIntervalRounding() {
+        Rounding tzRounding = Rounding.builder(TimeValue.timeValueHours(12)).build();
+        ZoneId tz = ZoneOffset.UTC;
+        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-03T00:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T00:00:00.000Z")), isDate(time("2009-02-03T12:00:00.000Z"), tz));
+        assertThat(tzRounding.round(time("2009-02-03T13:01:01")), isDate(time("2009-02-03T12:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T12:00:00.000Z")), isDate(time("2009-02-04T00:00:00.000Z"), tz));
+
+        tzRounding = Rounding.builder(TimeValue.timeValueHours(48)).build();
+        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-03T00:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T00:00:00.000Z")), isDate(time("2009-02-05T00:00:00.000Z"), tz));
+        assertThat(tzRounding.round(time("2009-02-05T13:01:01")), isDate(time("2009-02-05T00:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-05T00:00:00.000Z")), isDate(time("2009-02-07T00:00:00.000Z"), tz));
+    }
+
+    /**
+     * test TimeIntervalRounding, (interval &lt; 12h) with time zone shift
+     */
+    public void testTimeIntervalRounding() {
+        ZoneId tz = ZoneOffset.ofHours(-1);
+        Rounding tzRounding = Rounding.builder(TimeValue.timeValueHours(6)).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2009-02-03T00:01:01")), isDate(time("2009-02-02T19:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-02T19:00:00.000Z")), isDate(time("2009-02-03T01:00:00.000Z"), tz));
+
+        assertThat(tzRounding.round(time("2009-02-03T13:01:01")), isDate(time("2009-02-03T13:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T13:00:00.000Z")), isDate(time("2009-02-03T19:00:00.000Z"), tz));
+    }
+
+    /**
+     * test DayIntervalRounding, (interval &gt;= 12h) with time zone shift
+     */
+    public void testDayIntervalRounding() {
+        ZoneId tz = ZoneOffset.ofHours(-8);
+        Rounding tzRounding = Rounding.builder(TimeValue.timeValueHours(12)).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2009-02-03T00:01:01")), isDate(time("2009-02-02T20:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-02T20:00:00.000Z")), isDate(time("2009-02-03T08:00:00.000Z"), tz));
+
+        assertThat(tzRounding.round(time("2009-02-03T13:01:01")), isDate(time("2009-02-03T08:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T08:00:00.000Z")), isDate(time("2009-02-03T20:00:00.000Z"), tz));
+    }
+
+    public void testDayRounding() {
+        int timezoneOffset = -2;
+        Rounding tzRounding = Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH)
+            .timeZone(ZoneOffset.ofHours(timezoneOffset)).build();
+        assertThat(tzRounding.round(0), equalTo(0L - TimeValue.timeValueHours(24 + timezoneOffset).millis()));
+        assertThat(tzRounding.nextRoundingValue(0L - TimeValue.timeValueHours(24 + timezoneOffset).millis()), equalTo(0L - TimeValue
+            .timeValueHours(timezoneOffset).millis()));
+
+        ZoneId tz = ZoneId.of("-08:00");
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2012-04-01T04:15:30Z")), isDate(time("2012-03-31T08:00:00Z"), tz));
+
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.MONTH_OF_YEAR).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2012-04-01T04:15:30Z")), equalTo(time("2012-03-01T08:00:00Z")));
+
+        // date in Feb-3rd, but still in Feb-2nd in -02:00 timezone
+        tz = ZoneId.of("-02:00");
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-02T02:00:00"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-02T02:00:00")), isDate(time("2009-02-03T02:00:00"), tz));
+
+        // date in Feb-3rd, also in -02:00 timezone
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2009-02-03T02:01:01")), isDate(time("2009-02-03T02:00:00"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T02:00:00")), isDate(time("2009-02-04T02:00:00"), tz));
+    }
+
+    public void testTimeRounding() {
+        // hour unit
+        ZoneId tz = ZoneOffset.ofHours(-2);
+        Rounding tzRounding = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).timeZone(tz).build();
+        assertThat(tzRounding.round(0), equalTo(0L));
+        assertThat(tzRounding.nextRoundingValue(0L), equalTo(TimeValue.timeValueHours(1L).getMillis()));
+
+        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-03T01:00:00"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T01:00:00")), isDate(time("2009-02-03T02:00:00"), tz));
+    }
+
+    public void testTimeUnitRoundingDST() {
+        Rounding tzRounding;
+        // testing savings to non savings switch
+        ZoneId cet = ZoneId.of("CET");
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).timeZone(cet).build();
+        assertThat(tzRounding.round(time("2014-10-26T01:01:01", cet)), isDate(time("2014-10-26T01:00:00+02:00"), cet));
+        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T01:00:00", cet)),isDate(time("2014-10-26T02:00:00+02:00"), cet));
+        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T02:00:00", cet)), isDate(time("2014-10-26T02:00:00+01:00"), cet));
+
+        // testing non savings to savings switch
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).timeZone(cet).build();
+        assertThat(tzRounding.round(time("2014-03-30T01:01:01", cet)), isDate(time("2014-03-30T01:00:00+01:00"), cet));
+        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T01:00:00", cet)), isDate(time("2014-03-30T03:00:00", cet), cet));
+        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T03:00:00", cet)), isDate(time("2014-03-30T04:00:00", cet), cet));
+
+        // testing non savings to savings switch (America/Chicago)
+        ZoneId chg = ZoneId.of("America/Chicago");
+        Rounding tzRounding_utc = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY)
+            .timeZone(ZoneOffset.UTC).build();
+        assertThat(tzRounding.round(time("2014-03-09T03:01:01", chg)), isDate(time("2014-03-09T03:00:00", chg), chg));
+
+        Rounding tzRounding_chg = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).timeZone(chg).build();
+        assertThat(tzRounding_chg.round(time("2014-03-09T03:01:01", chg)), isDate(time("2014-03-09T03:00:00", chg), chg));
+
+        // testing savings to non savings switch 2013 (America/Chicago)
+        assertThat(tzRounding_utc.round(time("2013-11-03T06:01:01", chg)), isDate(time("2013-11-03T06:00:00", chg), chg));
+        assertThat(tzRounding_chg.round(time("2013-11-03T06:01:01", chg)), isDate(time("2013-11-03T06:00:00", chg), chg));
+
+        // testing savings to non savings switch 2014 (America/Chicago)
+        assertThat(tzRounding_utc.round(time("2014-11-02T06:01:01", chg)), isDate(time("2014-11-02T06:00:00", chg), chg));
+        assertThat(tzRounding_chg.round(time("2014-11-02T06:01:01", chg)), isDate(time("2014-11-02T06:00:00", chg), chg));
+    }
+
+    /**
+     * Randomized test on TimeUnitRounding. Test uses random
+     * {@link DateTimeUnit} and {@link ZoneId} and often (50% of the time)
+     * chooses test dates that are exactly on or close to offset changes (e.g.
+     * DST) in the chosen time zone.
+     *
+     * It rounds the test date down and up and performs various checks on the
+     * rounding unit interval that is defined by this. Assumptions tested are
+     * described in
+     * {@link #assertInterval(long, long, long, Rounding, ZoneId)}
+     */
+    public void testRoundingRandom() {
+        for (int i = 0; i < 1000; ++i) {
+            Rounding.DateTimeUnit unit = randomFrom(Rounding.DateTimeUnit.values());
+            ZoneId tz = randomZone();
+            Rounding rounding = new Rounding.TimeUnitRounding(unit, tz);
+            long date = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00
+            long unitMillis = unit.getField().getBaseUnit().getDuration().toMillis();
+            // FIXME this was copy pasted from the other impl and not used. breaks the nasty date actually gets assigned
+            if (randomBoolean()) {
+                nastyDate(date, tz, unitMillis);
+            }
+            final long roundedDate = rounding.round(date);
+            final long nextRoundingValue = rounding.nextRoundingValue(roundedDate);
+
+            assertInterval(roundedDate, date, nextRoundingValue, rounding, tz);
+
+            // check correct unit interval width for units smaller than a day, they should be fixed size except for transitions
+            if (unitMillis <= 86400 * 1000) {
+                // if the interval defined didn't cross timezone offset transition, it should cover unitMillis width
+                int offsetRounded = tz.getRules().getOffset(Instant.ofEpochMilli(roundedDate - 1)).getTotalSeconds();
+                int offsetNextValue = tz.getRules().getOffset(Instant.ofEpochMilli(nextRoundingValue + 1)).getTotalSeconds();
+                if (offsetRounded == offsetNextValue) {
+                    assertThat("unit interval width not as expected for [" + unit + "], [" + tz + "] at "
+                        + Instant.ofEpochMilli(roundedDate), nextRoundingValue - roundedDate, equalTo(unitMillis));
+                }
+            }
+        }
+    }
+
+    /**
+     * To be even more nasty, go to a transition in the selected time zone.
+     * In one third of the cases stay there, otherwise go half a unit back or forth
+     */
+    private static long nastyDate(long initialDate, ZoneId timezone, long unitMillis) {
+        ZoneOffsetTransition transition = timezone.getRules().nextTransition(Instant.ofEpochMilli(initialDate));
+        long date = initialDate;
+        if (transition != null) {
+            date = transition.getInstant().toEpochMilli();
+        }
+        if (randomBoolean()) {
+            return date + (randomLong() % unitMillis);  // positive and negative offset possible
+        } else {
+            return date;
+        }
+    }
+
+    /**
+     * test DST end with interval rounding
+     * CET: 25 October 2015, 03:00:00 clocks were turned backward 1 hour to 25 October 2015, 02:00:00 local standard time
+     */
+    public void testTimeIntervalCET_DST_End() {
+        long interval = TimeUnit.MINUTES.toMillis(20);
+        ZoneId tz = ZoneId.of("CET");
+        Rounding rounding = new Rounding.TimeIntervalRounding(interval, tz);
+
+        assertThat(rounding.round(time("2015-10-25T01:55:00+02:00")), isDate(time("2015-10-25T01:40:00+02:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T02:15:00+02:00")), isDate(time("2015-10-25T02:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T02:35:00+02:00")), isDate(time("2015-10-25T02:20:00+02:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T02:55:00+02:00")), isDate(time("2015-10-25T02:40:00+02:00"), tz));
+        // after DST shift
+        assertThat(rounding.round(time("2015-10-25T02:15:00+01:00")), isDate(time("2015-10-25T02:00:00+01:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T02:35:00+01:00")), isDate(time("2015-10-25T02:20:00+01:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T02:55:00+01:00")), isDate(time("2015-10-25T02:40:00+01:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T03:15:00+01:00")), isDate(time("2015-10-25T03:00:00+01:00"), tz));
+    }
+
+    /**
+     * test DST start with interval rounding
+     * CET: 27 March 2016, 02:00:00 clocks were turned forward 1 hour to 27 March 2016, 03:00:00 local daylight time
+     */
+    public void testTimeIntervalCET_DST_Start() {
+        long interval = TimeUnit.MINUTES.toMillis(20);
+        ZoneId tz = ZoneId.of("CET");
+        Rounding rounding = new Rounding.TimeIntervalRounding(interval, tz);
+        // test DST start
+        assertThat(rounding.round(time("2016-03-27T01:55:00+01:00")), isDate(time("2016-03-27T01:40:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T02:00:00+01:00")), isDate(time("2016-03-27T03:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:15:00+02:00")), isDate(time("2016-03-27T03:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:35:00+02:00")), isDate(time("2016-03-27T03:20:00+02:00"), tz));
+    }
+
+    /**
+     * test DST start with offset not fitting interval, e.g. Asia/Kathmandu
+     * adding 15min on 1986-01-01T00:00:00 the interval from
+     * 1986-01-01T00:15:00+05:45 to 1986-01-01T00:20:00+05:45 to only be 5min
+     * long
+     */
+    public void testTimeInterval_Kathmandu_DST_Start() {
+        long interval = TimeUnit.MINUTES.toMillis(20);
+        ZoneId tz = ZoneId.of("Asia/Kathmandu");
+        Rounding rounding = new Rounding.TimeIntervalRounding(interval, tz);
+        assertThat(rounding.round(time("1985-12-31T23:55:00+05:30")), isDate(time("1985-12-31T23:40:00+05:30"), tz));
+        assertThat(rounding.round(time("1986-01-01T00:16:00+05:45")), isDate(time("1986-01-01T00:15:00+05:45"), tz));
+        assertThat(time("1986-01-01T00:15:00+05:45") - time("1985-12-31T23:40:00+05:30"), equalTo(TimeUnit.MINUTES.toMillis(20)));
+        assertThat(rounding.round(time("1986-01-01T00:26:00+05:45")), isDate(time("1986-01-01T00:20:00+05:45"), tz));
+        assertThat(time("1986-01-01T00:20:00+05:45") - time("1986-01-01T00:15:00+05:45"), equalTo(TimeUnit.MINUTES.toMillis(5)));
+        assertThat(rounding.round(time("1986-01-01T00:46:00+05:45")), isDate(time("1986-01-01T00:40:00+05:45"), tz));
+        assertThat(time("1986-01-01T00:40:00+05:45") - time("1986-01-01T00:20:00+05:45"), equalTo(TimeUnit.MINUTES.toMillis(20)));
+    }
+
+    /**
+     * Special test for intervals that don't fit evenly into rounding interval.
+     * In this case, when interval crosses DST transition point, rounding in local
+     * time can land in a DST gap which results in wrong UTC rounding values.
+     */
+    public void testIntervalRounding_NotDivisibleInteval() {
+        long interval = TimeUnit.MINUTES.toMillis(14);
+        ZoneId tz = ZoneId.of("CET");
+        Rounding rounding = new Rounding.TimeIntervalRounding(interval, tz);
+
+        assertThat(rounding.round(time("2016-03-27T01:41:00+01:00")), isDate(time("2016-03-27T01:30:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T01:51:00+01:00")), isDate(time("2016-03-27T01:44:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T01:59:00+01:00")), isDate(time("2016-03-27T01:58:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:05:00+02:00")), isDate(time("2016-03-27T03:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:12:00+02:00")), isDate(time("2016-03-27T03:08:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:25:00+02:00")), isDate(time("2016-03-27T03:22:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:39:00+02:00")), isDate(time("2016-03-27T03:36:00+02:00"), tz));
+    }
+
+    /**
+     * Test for half day rounding intervals scrossing DST.
+     */
+    public void testIntervalRounding_HalfDay_DST() {
+        long interval = TimeUnit.HOURS.toMillis(12);
+        ZoneId tz = ZoneId.of("CET");
+        Rounding rounding = new Rounding.TimeIntervalRounding(interval, tz);
+
+        assertThat(rounding.round(time("2016-03-26T01:00:00+01:00")), isDate(time("2016-03-26T00:00:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-26T13:00:00+01:00")), isDate(time("2016-03-26T12:00:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T01:00:00+01:00")), isDate(time("2016-03-27T00:00:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T13:00:00+02:00")), isDate(time("2016-03-27T12:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-28T01:00:00+02:00")), isDate(time("2016-03-28T00:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-28T13:00:00+02:00")), isDate(time("2016-03-28T12:00:00+02:00"), tz));
+    }
+
+    /**
+     * randomized test on {@link org.elasticsearch.common.rounding.Rounding.TimeIntervalRounding} with random interval and time zone offsets
+     */
+    public void testIntervalRoundingRandom() {
+        for (int i = 0; i < 1000; i++) {
+            TimeUnit unit = randomFrom(TimeUnit.MINUTES, TimeUnit.HOURS, TimeUnit.DAYS);
+            long interval = unit.toMillis(randomIntBetween(1, 365));
+            ZoneId tz = randomZone();
+            Rounding rounding = new Rounding.TimeIntervalRounding(interval, tz);
+            long mainDate = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00
+            if (randomBoolean()) {
+                mainDate = nastyDate(mainDate, tz, interval);
+            }
+            // check two intervals around date
+            long previousRoundedValue = Long.MIN_VALUE;
+            for (long date = mainDate - 2 * interval; date < mainDate + 2 * interval; date += interval / 2) {
+                try {
+                    final long roundedDate = rounding.round(date);
+                    final long nextRoundingValue = rounding.nextRoundingValue(roundedDate);
+                    assertThat("Rounding should be idempotent", roundedDate, equalTo(rounding.round(roundedDate)));
+                    assertThat("Rounded value smaller or equal than unrounded", roundedDate, lessThanOrEqualTo(date));
+                    assertThat("Values smaller than rounded value should round further down", rounding.round(roundedDate - 1),
+                        lessThan(roundedDate));
+                    assertThat("Rounding should be >= previous rounding value", roundedDate, greaterThanOrEqualTo(previousRoundedValue));
+
+                    if (tz.getRules().isFixedOffset()) {
+                        assertThat("NextRounding value should be greater than date", nextRoundingValue, greaterThan(roundedDate));
+                        assertThat("NextRounding value should be interval from rounded value", nextRoundingValue - roundedDate,
+                            equalTo(interval));
+                        assertThat("NextRounding value should be a rounded date", nextRoundingValue,
+                            equalTo(rounding.round(nextRoundingValue)));
+                    }
+                    previousRoundedValue = roundedDate;
+                } catch (AssertionError e) {
+                    ZonedDateTime dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(date), tz);
+                    ZonedDateTime previousRoundedValueDate = ZonedDateTime.ofInstant(Instant.ofEpochMilli(previousRoundedValue), tz);
+                    logger.error("Rounding error at {}/{}, timezone {}, interval: {} previousRoundedValue {}/{}", dateTime, date,
+                        tz, interval, previousRoundedValueDate, previousRoundedValue);
+                    throw e;
+                }
+            }
+        }
+    }
+
+    /**
+     * Test that rounded values are always greater or equal to last rounded value if date is increasing.
+     * The example covers an interval around 2011-10-30T02:10:00+01:00, time zone CET, interval: 2700000ms
+     */
+    public void testIntervalRoundingMonotonic_CET() {
+        long interval = TimeUnit.MINUTES.toMillis(45);
+        ZoneId tz = ZoneId.of("CET");
+        Rounding rounding = new Rounding.TimeIntervalRounding(interval, tz);
+        List<Tuple<String, String>> expectedDates = new ArrayList<>();
+        // first date is the date to be rounded, second the expected result
+        expectedDates.add(new Tuple<>("2011-10-30T01:40:00.000+02:00", "2011-10-30T01:30:00.000+02:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:02:30.000+02:00", "2011-10-30T01:30:00.000+02:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:25:00.000+02:00", "2011-10-30T02:15:00.000+02:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:47:30.000+02:00", "2011-10-30T02:15:00.000+02:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:10:00.000+01:00", "2011-10-30T02:15:00.000+02:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:32:30.000+01:00", "2011-10-30T02:15:00.000+01:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:55:00.000+01:00", "2011-10-30T02:15:00.000+01:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T03:17:30.000+01:00", "2011-10-30T03:00:00.000+01:00"));
+
+        long previousDate = Long.MIN_VALUE;
+        for (Tuple<String, String> dates : expectedDates) {
+            final long roundedDate = rounding.round(time(dates.v1()));
+            assertThat(dates.toString(), roundedDate, isDate(time(dates.v2()), tz));
+            assertThat(dates.toString(), roundedDate, greaterThanOrEqualTo(previousDate));
+            previousDate = roundedDate;
+        }
+        // here's what this means for interval widths
+        assertEquals(TimeUnit.MINUTES.toMillis(45), time("2011-10-30T02:15:00.000+02:00") - time("2011-10-30T01:30:00.000+02:00"));
+        assertEquals(TimeUnit.MINUTES.toMillis(60), time("2011-10-30T02:15:00.000+01:00") - time("2011-10-30T02:15:00.000+02:00"));
+        assertEquals(TimeUnit.MINUTES.toMillis(45), time("2011-10-30T03:00:00.000+01:00") - time("2011-10-30T02:15:00.000+01:00"));
+    }
+
+    /**
+     * special test for DST switch from #9491
+     */
+    public void testAmbiguousHoursAfterDSTSwitch() {
+        Rounding tzRounding;
+        final ZoneId tz = ZoneId.of("Asia/Jerusalem");
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2014-10-26T00:30:00+03:00")), isDate(time("2014-10-26T00:00:00+03:00"), tz));
+        assertThat(tzRounding.round(time("2014-10-26T01:30:00+03:00")), isDate(time("2014-10-26T01:00:00+03:00"), tz));
+        // the utc date for "2014-10-25T03:00:00+03:00" and "2014-10-25T03:00:00+02:00" is the same, local time turns back 1h here
+        assertThat(time("2014-10-26T03:00:00+03:00"), isDate(time("2014-10-26T02:00:00+02:00"), tz));
+        assertThat(tzRounding.round(time("2014-10-26T01:30:00+02:00")), isDate(time("2014-10-26T01:00:00+02:00"), tz));
+        assertThat(tzRounding.round(time("2014-10-26T02:30:00+02:00")), isDate(time("2014-10-26T02:00:00+02:00"), tz));
+
+        // Day interval
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)), isDate(time("2014-11-11T00:00:00", tz), tz));
+        // DST on
+        assertThat(tzRounding.round(time("2014-08-11T17:00:00", tz)), isDate(time("2014-08-11T00:00:00", tz), tz));
+        // Day of switching DST on -> off
+        assertThat(tzRounding.round(time("2014-10-26T17:00:00", tz)), isDate(time("2014-10-26T00:00:00", tz), tz));
+        // Day of switching DST off -> on
+        assertThat(tzRounding.round(time("2015-03-27T17:00:00", tz)), isDate(time("2015-03-27T00:00:00", tz), tz));
+
+        // Month interval
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.MONTH_OF_YEAR).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)), isDate(time("2014-11-01T00:00:00", tz), tz));
+        // DST on
+        assertThat(tzRounding.round(time("2014-10-10T17:00:00", tz)), isDate(time("2014-10-01T00:00:00", tz), tz));
+
+        // Year interval
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.YEAR_OF_CENTURY).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)), isDate(time("2014-01-01T00:00:00", tz), tz));
+
+        // Two timestamps in same year and different timezone offset ("Double buckets" issue - #9491)
+        tzRounding = Rounding.builder(Rounding.DateTimeUnit.YEAR_OF_CENTURY).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)),
+            isDate(tzRounding.round(time("2014-08-11T17:00:00", tz)), tz));
+    }
+
+    /**
+     * test for #10025, strict local to UTC conversion can cause joda exceptions
+     * on DST start
+     */
+    public void testLenientConversionDST() {
+        ZoneId tz = ZoneId.of("America/Sao_Paulo");
+
+        long start = time("2014-10-18T20:50:00.000", tz);
+        long end = time("2014-10-19T01:00:00.000", tz);
+        Rounding tzRounding = new Rounding.TimeUnitRounding(Rounding.DateTimeUnit.MINUTES_OF_HOUR, tz);
+        Rounding dayTzRounding = new Rounding.TimeIntervalRounding(60000, tz);
+        for (long time = start; time < end; time = time + 60000) {
+            assertThat(tzRounding.nextRoundingValue(time), greaterThan(time));
+            assertThat(dayTzRounding.nextRoundingValue(time), greaterThan(time));
+        }
+    }
+
+    public void testEdgeCasesTransition() {
+        {
+            // standard +/-1 hour DST transition, CET
+            ZoneId tz = ZoneId.of("CET");
+            Rounding rounding = new Rounding.TimeUnitRounding(Rounding.DateTimeUnit.HOUR_OF_DAY, tz);
+
+            // 29 Mar 2015 - Daylight Saving Time Started
+            // at 02:00:00 clocks were turned forward 1 hour to 03:00:00
+            assertInterval(time("2015-03-29T00:00:00.000+01:00"), time("2015-03-29T01:00:00.000+01:00"), rounding, 60, tz);
+            assertInterval(time("2015-03-29T01:00:00.000+01:00"), time("2015-03-29T03:00:00.000+02:00"), rounding, 60, tz);
+            assertInterval(time("2015-03-29T03:00:00.000+02:00"), time("2015-03-29T04:00:00.000+02:00"), rounding, 60, tz);
+
+            // 25 Oct 2015 - Daylight Saving Time Ended
+            // at 03:00:00 clocks were turned backward 1 hour to 02:00:00
+            assertInterval(time("2015-10-25T01:00:00.000+02:00"), time("2015-10-25T02:00:00.000+02:00"), rounding, 60, tz);
+            assertInterval(time("2015-10-25T02:00:00.000+02:00"), time("2015-10-25T02:00:00.000+01:00"), rounding, 60, tz);
+            assertInterval(time("2015-10-25T02:00:00.000+01:00"), time("2015-10-25T03:00:00.000+01:00"), rounding, 60, tz);
+        }
+
+        {
+            // time zone "Asia/Kathmandu"
+            // 1 Jan 1986 - Time Zone Change (IST → NPT), at 00:00:00 clocks were turned forward 00:15 minutes
+            //
+            // hour rounding is stable before 1985-12-31T23:00:00.000 and after 1986-01-01T01:00:00.000+05:45
+            // the interval between is 105 minutes long because the hour after transition starts at 00:15
+            // which is not a round value for hourly rounding
+            ZoneId tz = ZoneId.of("Asia/Kathmandu");
+            Rounding rounding = new Rounding.TimeUnitRounding(Rounding.DateTimeUnit.HOUR_OF_DAY, tz);
+
+            assertInterval(time("1985-12-31T22:00:00.000+05:30"), time("1985-12-31T23:00:00.000+05:30"), rounding, 60, tz);
+            assertInterval(time("1985-12-31T23:00:00.000+05:30"), time("1986-01-01T01:00:00.000+05:45"), rounding, 105, tz);
+            assertInterval(time("1986-01-01T01:00:00.000+05:45"), time("1986-01-01T02:00:00.000+05:45"), rounding, 60, tz);
+        }
+
+        {
+            // time zone "Australia/Lord_Howe"
+            // 3 Mar 1991 - Daylight Saving Time Ended
+            // at 02:00:00 clocks were turned backward 0:30 hours to Sunday, 3 March 1991, 01:30:00
+            ZoneId tz = ZoneId.of("Australia/Lord_Howe");
+            Rounding rounding = new Rounding.TimeUnitRounding(Rounding.DateTimeUnit.HOUR_OF_DAY, tz);
+
+            assertInterval(time("1991-03-03T00:00:00.000+11:00"), time("1991-03-03T01:00:00.000+11:00"), rounding, 60, tz);
+            assertInterval(time("1991-03-03T01:00:00.000+11:00"), time("1991-03-03T02:00:00.000+10:30"), rounding, 90, tz);
+            assertInterval(time("1991-03-03T02:00:00.000+10:30"), time("1991-03-03T03:00:00.000+10:30"), rounding, 60, tz);
+
+            // 27 Oct 1991 - Daylight Saving Time Started
+            // at 02:00:00 clocks were turned forward 0:30 hours to 02:30:00
+            assertInterval(time("1991-10-27T00:00:00.000+10:30"), time("1991-10-27T01:00:00.000+10:30"), rounding, 60, tz);
+            // the interval containing the switch time is 90 minutes long
+            assertInterval(time("1991-10-27T01:00:00.000+10:30"), time("1991-10-27T03:00:00.000+11:00"), rounding, 90, tz);
+            assertInterval(time("1991-10-27T03:00:00.000+11:00"), time("1991-10-27T04:00:00.000+11:00"), rounding, 60, tz);
+        }
+
+        {
+            // time zone "Pacific/Chatham"
+            // 5 Apr 2015 - Daylight Saving Time Ended
+            // at 03:45:00 clocks were turned backward 1 hour to 02:45:00
+            ZoneId tz = ZoneId.of("Pacific/Chatham");
+            Rounding rounding = new Rounding.TimeUnitRounding(Rounding.DateTimeUnit.HOUR_OF_DAY, tz);
+
+            assertInterval(time("2015-04-05T02:00:00.000+13:45"), time("2015-04-05T03:00:00.000+13:45"), rounding, 60, tz);
+            assertInterval(time("2015-04-05T03:00:00.000+13:45"), time("2015-04-05T03:00:00.000+12:45"), rounding, 60, tz);
+            assertInterval(time("2015-04-05T03:00:00.000+12:45"), time("2015-04-05T04:00:00.000+12:45"), rounding, 60, tz);
+
+            // 27 Sep 2015 - Daylight Saving Time Started
+            // at 02:45:00 clocks were turned forward 1 hour to 03:45:00
+
+            assertInterval(time("2015-09-27T01:00:00.000+12:45"), time("2015-09-27T02:00:00.000+12:45"), rounding, 60, tz);
+            assertInterval(time("2015-09-27T02:00:00.000+12:45"), time("2015-09-27T04:00:00.000+13:45"), rounding, 60, tz);
+            assertInterval(time("2015-09-27T04:00:00.000+13:45"), time("2015-09-27T05:00:00.000+13:45"), rounding, 60, tz);
+        }
+    }
+
+    public void testDST_Europe_Rome() {
+        // time zone "Europe/Rome", rounding to days. Rome had two midnights on the day the clocks went back in 1978, and
+        // timeZone.convertLocalToUTC() gives the later of the two because Rome is east of UTC, whereas we want the earlier.
+
+        ZoneId tz = ZoneId.of("Europe/Rome");
+        Rounding rounding = new Rounding.TimeUnitRounding(Rounding.DateTimeUnit.DAY_OF_MONTH, tz);
+
+        {
+            long timeBeforeFirstMidnight = time("1978-09-30T23:59:00+02:00");
+            long floor = rounding.round(timeBeforeFirstMidnight);
+            assertThat(floor, isDate(time("1978-09-30T00:00:00+02:00"), tz));
+        }
+
+        {
+            long timeBetweenMidnights = time("1978-10-01T00:30:00+02:00");
+            long floor = rounding.round(timeBetweenMidnights);
+            assertThat(floor, isDate(time("1978-10-01T00:00:00+02:00"), tz));
+        }
+
+        {
+            long timeAfterSecondMidnight = time("1978-10-01T00:30:00+01:00");
+            long floor = rounding.round(timeAfterSecondMidnight);
+            assertThat(floor, isDate(time("1978-10-01T00:00:00+02:00"), tz));
+
+            long prevFloor = rounding.round(floor - 1);
+            assertThat(prevFloor, lessThan(floor));
+            assertThat(prevFloor, isDate(time("1978-09-30T00:00:00+02:00"), tz));
+        }
+    }
+
+    /**
+     * Test for a time zone whose days overlap because the clocks are set back across midnight at the end of DST.
+     */
+    public void testDST_America_St_Johns() {
+        // time zone "America/St_Johns", rounding to days.
+        ZoneId tz = ZoneId.of("America/St_Johns");
+        Rounding rounding = new Rounding.TimeUnitRounding(Rounding.DateTimeUnit.DAY_OF_MONTH, tz);
+
+        // 29 October 2006 - Daylight Saving Time ended, changing the UTC offset from -02:30 to -03:30.
+        // This happened at 02:31 UTC, 00:01 local time, so the clocks were set back 1 hour to 23:01 on the 28th.
+        // This means that 2006-10-29 has _two_ midnights, one in the -02:30 offset and one in the -03:30 offset.
+        // Only the first of these is considered "rounded". Moreover, the extra time between 23:01 and 23:59
+        // should be considered as part of the 28th even though it comes after midnight on the 29th.
+
+        {
+            // Times before the first midnight should be rounded up to the first midnight.
+            long timeBeforeFirstMidnight = time("2006-10-28T23:30:00.000-02:30");
+            long floor = rounding.round(timeBeforeFirstMidnight);
+            assertThat(floor, isDate(time("2006-10-28T00:00:00.000-02:30"), tz));
+            long ceiling = rounding.nextRoundingValue(timeBeforeFirstMidnight);
+            assertThat(ceiling, isDate(time("2006-10-29T00:00:00.000-02:30"), tz));
+            assertInterval(floor, timeBeforeFirstMidnight, ceiling, rounding, tz);
+        }
+
+        {
+            // Times between the two midnights which are on the later day should be rounded down to the later day's midnight.
+            long timeBetweenMidnights = time("2006-10-29T00:00:30.000-02:30");
+            // (this is halfway through the last minute before the clocks changed, in which local time was ambiguous)
+
+            long floor = rounding.round(timeBetweenMidnights);
+            assertThat(floor, isDate(time("2006-10-29T00:00:00.000-02:30"), tz));
+
+            long ceiling = rounding.nextRoundingValue(timeBetweenMidnights);
+            assertThat(ceiling, isDate(time("2006-10-30T00:00:00.000-03:30"), tz));
+
+            assertInterval(floor, timeBetweenMidnights, ceiling, rounding, tz);
+        }
+
+        {
+            // Times between the two midnights which are on the earlier day should be rounded down to the earlier day's midnight.
+            long timeBetweenMidnights = time("2006-10-28T23:30:00.000-03:30");
+            // (this is halfway through the hour after the clocks changed, in which local time was ambiguous)
+
+            long floor = rounding.round(timeBetweenMidnights);
+            assertThat(floor, isDate(time("2006-10-28T00:00:00.000-02:30"), tz));
+
+            long ceiling = rounding.nextRoundingValue(timeBetweenMidnights);
+            assertThat(ceiling, isDate(time("2006-10-29T00:00:00.000-02:30"), tz));
+
+            assertInterval(floor, timeBetweenMidnights, ceiling, rounding, tz);
+        }
+
+        {
+            // Times after the second midnight should be rounded down to the first midnight.
+            long timeAfterSecondMidnight = time("2006-10-29T06:00:00.000-03:30");
+            long floor = rounding.round(timeAfterSecondMidnight);
+            assertThat(floor, isDate(time("2006-10-29T00:00:00.000-02:30"), tz));
+            long ceiling = rounding.nextRoundingValue(timeAfterSecondMidnight);
+            assertThat(ceiling, isDate(time("2006-10-30T00:00:00.000-03:30"), tz));
+            assertInterval(floor, timeAfterSecondMidnight, ceiling, rounding, tz);
+        }
+    }
+
+    /**
+     * tests for dst transition with overlaps and day roundings.
+     */
+    public void testDST_END_Edgecases() {
+        // First case, dst happens at 1am local time, switching back one hour.
+        // We want the overlapping hour to count for the next day, making it a 25h interval
+
+        ZoneId tz = ZoneId.of("Atlantic/Azores");
+        Rounding.DateTimeUnit timeUnit = Rounding.DateTimeUnit.DAY_OF_MONTH;
+        Rounding rounding = new Rounding.TimeUnitRounding(timeUnit, tz);
+
+        // Sunday, 29 October 2000, 01:00:00 clocks were turned backward 1 hour
+        // to Sunday, 29 October 2000, 00:00:00 local standard time instead
+        // which means there were two midnights that day.
+
+        long midnightBeforeTransition = time("2000-10-29T00:00:00", tz);
+        long midnightOfTransition = time("2000-10-29T00:00:00-01:00");
+        assertEquals(60L * 60L * 1000L, midnightOfTransition - midnightBeforeTransition);
+        long nextMidnight = time("2000-10-30T00:00:00", tz);
+
+        assertInterval(midnightBeforeTransition, nextMidnight, rounding, 25 * 60, tz);
+
+        assertThat(rounding.round(time("2000-10-29T06:00:00-01:00")), isDate(time("2000-10-29T00:00:00Z"), tz));
+
+        // Second case, dst happens at 0am local time, switching back one hour to 23pm local time.
+        // We want the overlapping hour to count for the previous day here
+
+        tz = ZoneId.of("America/Lima");
+        rounding = new Rounding.TimeUnitRounding(timeUnit, tz);
+
+        // Sunday, 1 April 1990, 00:00:00 clocks were turned backward 1 hour to
+        // Saturday, 31 March 1990, 23:00:00 local standard time instead
+
+        midnightBeforeTransition = time("1990-03-31T00:00:00.000-04:00");
+        nextMidnight = time("1990-04-01T00:00:00.000-05:00");
+        assertInterval(midnightBeforeTransition, nextMidnight, rounding, 25 * 60, tz);
+
+        // make sure the next interval is 24h long again
+        long midnightAfterTransition = time("1990-04-01T00:00:00.000-05:00");
+        nextMidnight = time("1990-04-02T00:00:00.000-05:00");
+        assertInterval(midnightAfterTransition, nextMidnight, rounding, 24 * 60, tz);
+    }
+
+    private void assertInterval(long rounded, long nextRoundingValue, Rounding rounding, int minutes,
+                                ZoneId tz) {
+        assertInterval(rounded, dateBetween(rounded, nextRoundingValue), nextRoundingValue, rounding, tz);
+        long millisPerMinute = 60_000;
+        assertEquals(millisPerMinute * minutes, nextRoundingValue - rounded);
+    }
+
+    /**
+     * perform a number on assertions and checks on {@link org.elasticsearch.common.rounding.Rounding.TimeUnitRounding} intervals
+     * @param rounded the expected low end of the rounding interval
+     * @param unrounded a date in the interval to be checked for rounding
+     * @param nextRoundingValue the expected upper end of the rounding interval
+     * @param rounding the rounding instance
+     */
+    private void assertInterval(long rounded, long unrounded, long nextRoundingValue, Rounding rounding, ZoneId tz) {
+        assertThat("rounding should be idempotent ", rounding.round(rounded), isDate(rounded, tz));
+        assertThat("rounded value smaller or equal than unrounded" + rounding, rounded, lessThanOrEqualTo(unrounded));
+        assertThat("values less than rounded should round further down" + rounding, rounding.round(rounded - 1), lessThan(rounded));
+        assertThat("nextRounding value should be a rounded date", rounding.round(nextRoundingValue), isDate(nextRoundingValue, tz));
+        assertThat("values above nextRounding should round down there", rounding.round(nextRoundingValue + 1),
+            isDate(nextRoundingValue, tz));
+
+        if (isTimeWithWellDefinedRounding(tz, unrounded)) {
+            assertThat("nextRounding value should be greater than date" + rounding, nextRoundingValue, greaterThan(unrounded));
+
+            long dateBetween = dateBetween(rounded, nextRoundingValue);
+            long roundingDateBetween = rounding.round(dateBetween);
+            ZonedDateTime zonedDateBetween = ZonedDateTime.ofInstant(Instant.ofEpochMilli(dateBetween), tz);
+            assertThat("dateBetween [" + zonedDateBetween + "/" + dateBetween + "] should round down to roundedDate [" +
+                    Instant.ofEpochMilli(roundingDateBetween) + "]", roundingDateBetween, isDate(rounded, tz));
+            assertThat("dateBetween [" + zonedDateBetween + "] should round up to nextRoundingValue",
+                rounding.nextRoundingValue(dateBetween), isDate(nextRoundingValue, tz));
+        }
+    }
+
+    private static boolean isTimeWithWellDefinedRounding(ZoneId tz, long t) {
+        if (tz.getId().equals("America/St_Johns")
+            || tz.getId().equals("America/Goose_Bay")
+            || tz.getId().equals("America/Moncton")
+            || tz.getId().equals("Canada/Newfoundland")) {
+
+            // Clocks went back at 00:01 between 1987 and 2010, causing overlapping days.
+            // These timezones are otherwise uninteresting, so just skip this period.
+
+            return t <= time("1987-10-01T00:00:00Z")
+                || t >= time("2010-12-01T00:00:00Z");
+        }
+
+        if (tz.getId().equals("Antarctica/Casey")) {
+
+            // Clocks went back 3 hours at 02:00 on 2010-03-05, causing overlapping days.
+
+            return t <= time("2010-03-03T00:00:00Z")
+                || t >= time("2010-03-07T00:00:00Z");
+        }
+
+        return true;
+    }
+
+    private static long dateBetween(long lower, long upper) {
+        long dateBetween = randomLongBetween(lower, upper - 1);
+        assert lower <= dateBetween && dateBetween < upper;
+        return dateBetween;
+    }
+
+    private static long time(String time) {
+        return time(time, ZoneOffset.UTC);
+    }
+
+    private static long time(String time, ZoneId zone) {
+        TemporalAccessor accessor = DateFormatters.forPattern("date_optional_time").withZone(zone).parse(time);
+        return DateFormatters.toZonedDateTime(accessor).toInstant().toEpochMilli();
+    }
+
+    private static Matcher<Long> isDate(final long expected, ZoneId tz) {
+        return new TypeSafeMatcher<Long>() {
+            @Override
+            public boolean matchesSafely(final Long item) {
+                return expected == item;
+            }
+
+            @Override
+            public void describeTo(Description description) {
+                ZonedDateTime zonedDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(expected), tz);
+                description.appendText(DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(zonedDateTime) + " [" + expected + "] ");
+            }
+
+            @Override
+            protected void describeMismatchSafely(final Long actual, final Description mismatchDescription) {
+                ZonedDateTime zonedDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(actual), tz);
+                mismatchDescription.appendText(" was ")
+                    .appendValue(DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(zonedDateTime) + " [" + actual + "]");
+            }
+        };
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java b/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
index 7f8b893caf085..83bf2bae6a66f 100644
--- a/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
@@ -143,4 +143,22 @@ public void testHoleThatIsNorthOfPolygon() {
 
         assertEquals("Hole lies outside shell at or near point (4.0, 3.0, NaN)", e.getMessage());
     }
+
+    public void testWidePolygonWithConfusingOrientation() {
+        // A valid polygon that is oriented correctly (anticlockwise) but which
+        // confounds a naive algorithm for determining its orientation leading
+        // ES to believe that it crosses the dateline and "fixing" it in a way
+        // that self-intersects.
+
+        PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder()
+            .coordinate(10, -20).coordinate(100, 0).coordinate(-100, 0).coordinate(20, -45).coordinate(40, -60).close());
+        pb.build(); // Should not throw an exception
+    }
+
+    public void testPolygonWithUndefinedOrientationDueToCollinearPoints() {
+        PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder()
+            .coordinate(0.0, 0.0).coordinate(1.0, 1.0).coordinate(-1.0, -1.0).close());
+        InvalidShapeException e = expectThrows(InvalidShapeException.class, pb::build);
+        assertEquals("Cannot determine orientation: edges adjacent to (-1.0,-1.0) coincide", e.getMessage());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java b/server/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java
index ca9a6b3a1abf8..2fad9738cb59e 100644
--- a/server/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java
+++ b/server/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java
@@ -208,7 +208,13 @@ public void testExplicitRounding() {
         assertDateMathEquals("2014-11-18||/M", "2014-10-31T23:00:00.000Z", 0, false, DateTimeZone.forID("CET"));
         assertDateMathEquals("2014-11-18||/M", "2014-11-30T22:59:59.999Z", 0, true, DateTimeZone.forID("CET"));
 
+        assertDateMathEquals("2014-11-17T14||/w", "2014-11-17", 0, false, null);
         assertDateMathEquals("2014-11-18T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-19T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-20T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-21T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-22T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-23T14||/w", "2014-11-17", 0, false, null);
         assertDateMathEquals("2014-11-18T14||/w", "2014-11-23T23:59:59.999", 0, true, null);
         assertDateMathEquals("2014-11-18||/w", "2014-11-17", 0, false, null);
         assertDateMathEquals("2014-11-18||/w", "2014-11-23T23:59:59.999", 0, true, null);
diff --git a/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java b/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java
index d6f733d7c1cd4..1551a315b26d9 100644
--- a/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java
+++ b/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java
@@ -94,6 +94,7 @@ public void testDuellingFormatsValidParsing() {
 
         assertSameDate("2018-12-31", "date");
         assertSameDate("18-5-6", "date");
+        assertSameDate("10000-5-6", "date");
 
         assertSameDate("2018-12-31T12", "date_hour");
         assertSameDate("2018-12-31T8", "date_hour");
@@ -109,7 +110,16 @@ public void testDuellingFormatsValidParsing() {
         assertSameDate("2018-12-31T12:12:12.1", "date_hour_minute_second_millis");
         assertSameDate("2018-12-31T12:12:12.1", "date_hour_minute_second_fraction");
 
-        assertSameDate("2018-12-31", "date_optional_time");
+        assertSameDate("10000", "date_optional_time");
+        assertSameDate("10000T", "date_optional_time");
+        assertSameDate("2018", "date_optional_time");
+        assertSameDate("2018T", "date_optional_time");
+        assertSameDate("2018-05", "date_optional_time");
+        assertSameDate("2018-05-30", "date_optional_time");
+        assertSameDate("2018-05-30T20", "date_optional_time");
+        assertSameDate("2018-05-30T20:21", "date_optional_time");
+        assertSameDate("2018-05-30T20:21:23", "date_optional_time");
+        assertSameDate("2018-05-30T20:21:23.123", "date_optional_time");
         assertSameDate("2018-12-1", "date_optional_time");
         assertSameDate("2018-12-31T10:15:30", "date_optional_time");
         assertSameDate("2018-12-31T10:15:3", "date_optional_time");
@@ -230,6 +240,7 @@ public void testDuelingStrictParsing() {
         assertParseException("2018W313T81212Z", "strict_basic_week_date_time_no_millis");
         assertParseException("2018W313T12812Z", "strict_basic_week_date_time_no_millis");
         assertSameDate("2018-12-31", "strict_date");
+        assertParseException("10000-12-31", "strict_date");
         assertParseException("2018-8-31", "strict_date");
         assertSameDate("2018-12-31T12", "strict_date_hour");
         assertParseException("2018-12-31T8", "strict_date_hour");
@@ -246,6 +257,7 @@ public void testDuelingStrictParsing() {
         assertSameDate("2018-12-31", "strict_date_optional_time");
         assertParseException("2018-12-1", "strict_date_optional_time");
         assertParseException("2018-1-31", "strict_date_optional_time");
+        assertParseException("10000-01-31", "strict_date_optional_time");
         assertSameDate("2018-12-31T10:15:30", "strict_date_optional_time");
         assertParseException("2018-12-31T10:15:3", "strict_date_optional_time");
         assertParseException("2018-12-31T10:5:30", "strict_date_optional_time");
@@ -453,10 +465,12 @@ public void testSeveralTimeFormats() {
     }
 
     private void assertSamePrinterOutput(String format, ZonedDateTime javaDate, DateTime jodaDate) {
-        assertThat(jodaDate.getMillis(), is(javaDate.toEpochSecond() * 1000));
-        String javaTimeOut = DateFormatters.forPattern("dateOptionalTime").format(javaDate);
-        String jodaTimeOut = Joda.forPattern("dateOptionalTime").printer().print(jodaDate);
-        assertThat(javaTimeOut, is(jodaTimeOut));
+        assertThat(jodaDate.getMillis(), is(javaDate.toInstant().toEpochMilli()));
+        String javaTimeOut = DateFormatters.forPattern(format).format(javaDate);
+        String jodaTimeOut = Joda.forPattern(format).printer().print(jodaDate);
+        String message = String.format(Locale.ROOT, "expected string representation to be equal for format [%s]: joda [%s], java [%s]",
+                format, jodaTimeOut, javaTimeOut);
+        assertThat(message, javaTimeOut, is(jodaTimeOut));
     }
 
     private void assertSameDate(String input, String format) {
diff --git a/server/src/test/java/org/elasticsearch/common/logging/LoggersTests.java b/server/src/test/java/org/elasticsearch/common/logging/LoggersTests.java
index 6c18bd0afab1b..9b69a876c1d2a 100644
--- a/server/src/test/java/org/elasticsearch/common/logging/LoggersTests.java
+++ b/server/src/test/java/org/elasticsearch/common/logging/LoggersTests.java
@@ -46,7 +46,7 @@ static class MockAppender extends AbstractAppender {
 
         @Override
         public void append(LogEvent event) {
-            lastEvent = event;
+            lastEvent = event.toImmutable();
         }
 
         ParameterizedMessage lastParameterizedMessage() {
diff --git a/server/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java b/server/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java
index 6c92d3d1f9a19..33ea83c759297 100644
--- a/server/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java
+++ b/server/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java
@@ -19,6 +19,11 @@
 package org.elasticsearch.common.rounding;
 
 import org.elasticsearch.test.ESTestCase;
+import org.joda.time.DateTimeZone;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 
 import static org.elasticsearch.common.rounding.DateTimeUnit.DAY_OF_MONTH;
 import static org.elasticsearch.common.rounding.DateTimeUnit.HOUR_OF_DAY;
@@ -28,6 +33,7 @@
 import static org.elasticsearch.common.rounding.DateTimeUnit.SECOND_OF_MINUTE;
 import static org.elasticsearch.common.rounding.DateTimeUnit.WEEK_OF_WEEKYEAR;
 import static org.elasticsearch.common.rounding.DateTimeUnit.YEAR_OF_CENTURY;
+import static org.hamcrest.Matchers.is;
 
 public class DateTimeUnitTests extends ESTestCase {
 
@@ -59,4 +65,17 @@ public void testEnumIds() {
         assertEquals(8, SECOND_OF_MINUTE.id());
         assertEquals(SECOND_OF_MINUTE, DateTimeUnit.resolve((byte) 8));
     }
+
+    public void testConversion() {
+        long millis = randomLongBetween(0, Instant.now().toEpochMilli());
+        DateTimeZone zone = randomDateTimeZone();
+        ZoneId zoneId = ZoneId.of(zone.getID());
+
+        int offsetSeconds = zoneId.getRules().getOffset(Instant.ofEpochMilli(millis)).getTotalSeconds();
+        long parsedMillisJavaTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), zoneId)
+            .minusSeconds(offsetSeconds).toInstant().toEpochMilli();
+
+        long parsedMillisJodaTime = zone.convertLocalToUTC(millis, true);
+        assertThat(parsedMillisJavaTime, is(parsedMillisJodaTime));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/rounding/RoundingDuelTests.java b/server/src/test/java/org/elasticsearch/common/rounding/RoundingDuelTests.java
new file mode 100644
index 0000000000000..7e3dbdd5b94df
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/rounding/RoundingDuelTests.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.rounding;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.time.ZoneOffset;
+
+import static org.hamcrest.Matchers.is;
+
+public class RoundingDuelTests extends ESTestCase  {
+
+    // dont include nano/micro seconds as rounding would become zero then and throw an exception
+    private static final String[] ALLOWED_TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m"};
+
+    public void testSerialization() throws Exception {
+        org.elasticsearch.common.Rounding.DateTimeUnit randomDateTimeUnit =
+            randomFrom(org.elasticsearch.common.Rounding.DateTimeUnit.values());
+        org.elasticsearch.common.Rounding rounding;
+        if (randomBoolean()) {
+            rounding = org.elasticsearch.common.Rounding.builder(randomDateTimeUnit).timeZone(ZoneOffset.UTC).build();
+        } else {
+            rounding = org.elasticsearch.common.Rounding.builder(timeValue()).timeZone(ZoneOffset.UTC).build();
+        }
+        BytesStreamOutput output = new BytesStreamOutput();
+        rounding.writeTo(output);
+
+        Rounding roundingJoda = Rounding.Streams.read(output.bytes().streamInput());
+        org.elasticsearch.common.Rounding roundingJavaTime =
+            org.elasticsearch.common.Rounding.read(output.bytes().streamInput());
+
+        int randomInt = randomIntBetween(1, 1_000_000_000);
+        assertThat(roundingJoda.round(randomInt), is(roundingJavaTime.round(randomInt)));
+        assertThat(roundingJoda.nextRoundingValue(randomInt), is(roundingJavaTime.nextRoundingValue(randomInt)));
+    }
+
+    static TimeValue timeValue() {
+        return TimeValue.parseTimeValue(randomIntBetween(1, 1000) + randomFrom(ALLOWED_TIME_SUFFIXES), "settingName");
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
index fe7b02d63ecce..bb2b1df7f8c03 100644
--- a/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
+++ b/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
@@ -99,6 +99,15 @@ public void testCreate() throws Exception {
         assertTrue(keystore.getSettingNames().contains(KeyStoreWrapper.SEED_SETTING.getKey()));
     }
 
+    public void testDecryptKeyStoreWithWrongPassword() throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.create();
+        keystore.save(env.configFile(), new char[0]);
+        final KeyStoreWrapper loadedkeystore = KeyStoreWrapper.load(env.configFile());
+        final SecurityException exception = expectThrows(SecurityException.class,
+            () -> loadedkeystore.decrypt(new char[]{'i', 'n', 'v', 'a', 'l', 'i', 'd'}));
+        assertThat(exception.getMessage(), containsString("Keystore has been corrupted or tampered with"));
+    }
+
     public void testCannotReadStringFromClosedKeystore() throws Exception {
         KeyStoreWrapper keystore = KeyStoreWrapper.create();
         assertThat(keystore.getSettingNames(), Matchers.hasItem(KeyStoreWrapper.SEED_SETTING.getKey()));
diff --git a/server/src/test/java/org/elasticsearch/common/time/DateMathParserTests.java b/server/src/test/java/org/elasticsearch/common/time/DateMathParserTests.java
new file mode 100644
index 0000000000000..d6dc7bb36f2a6
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/time/DateMathParserTests.java
@@ -0,0 +1,309 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.time;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.test.ESTestCase;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.LongSupplier;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+
+public class DateMathParserTests extends ESTestCase {
+
+    private final CompoundDateTimeFormatter formatter = DateFormatters.forPattern("dateOptionalTime||epoch_millis");
+    private final DateMathParser parser = new DateMathParser(formatter);
+
+    public void testBasicDates() {
+        assertDateMathEquals("2014", "2014-01-01T00:00:00.000");
+        assertDateMathEquals("2014-05", "2014-05-01T00:00:00.000");
+        assertDateMathEquals("2014-05-30", "2014-05-30T00:00:00.000");
+        assertDateMathEquals("2014-05-30T20", "2014-05-30T20:00:00.000");
+        assertDateMathEquals("2014-05-30T20:21", "2014-05-30T20:21:00.000");
+        assertDateMathEquals("2014-05-30T20:21:35", "2014-05-30T20:21:35.000");
+        assertDateMathEquals("2014-05-30T20:21:35.123", "2014-05-30T20:21:35.123");
+    }
+
+    public void testRoundingDoesNotAffectExactDate() {
+        assertDateMathEquals("2014-11-12T22:55:00.000Z", "2014-11-12T22:55:00.000Z", 0, true, null);
+        assertDateMathEquals("2014-11-12T22:55:00.000Z", "2014-11-12T22:55:00.000Z", 0, false, null);
+
+        assertDateMathEquals("2014-11-12T22:55:00.000", "2014-11-12T21:55:00.000Z", 0, true, ZoneId.of("+01:00"));
+        assertDateMathEquals("2014-11-12T22:55:00.000", "2014-11-12T21:55:00.000Z", 0, false, ZoneId.of("+01:00"));
+
+        assertDateMathEquals("2014-11-12T22:55:00.000+01:00", "2014-11-12T21:55:00.000Z", 0, true, null);
+        assertDateMathEquals("2014-11-12T22:55:00.000+01:00", "2014-11-12T21:55:00.000Z", 0, false, null);
+    }
+
+    public void testTimezone() {
+        // timezone works within date format
+        assertDateMathEquals("2014-05-30T20:21+02:00", "2014-05-30T18:21:00.000");
+
+        // test alternative ways of writing zero offsets, according to ISO 8601 +00:00, +00, +0000 should work.
+        // joda also seems to allow for -00:00, -00, -0000
+        assertDateMathEquals("2014-05-30T18:21+00:00", "2014-05-30T18:21:00.000");
+        assertDateMathEquals("2014-05-30T18:21+00", "2014-05-30T18:21:00.000");
+        assertDateMathEquals("2014-05-30T18:21+0000", "2014-05-30T18:21:00.000");
+        assertDateMathEquals("2014-05-30T18:21-00:00", "2014-05-30T18:21:00.000");
+        assertDateMathEquals("2014-05-30T18:21-00", "2014-05-30T18:21:00.000");
+        assertDateMathEquals("2014-05-30T18:21-0000", "2014-05-30T18:21:00.000");
+
+        // but also externally
+        assertDateMathEquals("2014-05-30T20:21", "2014-05-30T18:21:00.000", 0, false, ZoneId.of("+02:00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, ZoneId.of("+00:00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, ZoneId.of("+00:00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, ZoneId.of("+00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, ZoneId.of("+0000"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, ZoneId.of("-00:00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, ZoneId.of("-00"));
+        assertDateMathEquals("2014-05-30T18:21", "2014-05-30T18:21:00.000", 0, false, ZoneId.of("-0000"));
+
+        // and timezone in the date has priority
+        assertDateMathEquals("2014-05-30T20:21+03:00", "2014-05-30T17:21:00.000", 0, false, ZoneId.of("-08:00"));
+        assertDateMathEquals("2014-05-30T20:21Z", "2014-05-30T20:21:00.000", 0, false, ZoneId.of("-08:00"));
+    }
+
+    public void testBasicMath() {
+        assertDateMathEquals("2014-11-18||+y", "2015-11-18");
+        assertDateMathEquals("2014-11-18||-2y", "2012-11-18");
+
+        assertDateMathEquals("2014-11-18||+3M", "2015-02-18");
+        assertDateMathEquals("2014-11-18||-M", "2014-10-18");
+
+        assertDateMathEquals("2014-11-18||+1w", "2014-11-25");
+        assertDateMathEquals("2014-11-18||-3w", "2014-10-28");
+
+        assertDateMathEquals("2014-11-18||+22d", "2014-12-10");
+        assertDateMathEquals("2014-11-18||-423d", "2013-09-21");
+
+        assertDateMathEquals("2014-11-18T14||+13h", "2014-11-19T03");
+        assertDateMathEquals("2014-11-18T14||-1h", "2014-11-18T13");
+        assertDateMathEquals("2014-11-18T14||+13H", "2014-11-19T03");
+        assertDateMathEquals("2014-11-18T14||-1H", "2014-11-18T13");
+
+        assertDateMathEquals("2014-11-18T14:27||+10240m", "2014-11-25T17:07");
+        assertDateMathEquals("2014-11-18T14:27||-10m", "2014-11-18T14:17");
+
+        assertDateMathEquals("2014-11-18T14:27:32||+60s", "2014-11-18T14:28:32");
+        assertDateMathEquals("2014-11-18T14:27:32||-3600s", "2014-11-18T13:27:32");
+    }
+
+    public void testLenientEmptyMath() {
+        assertDateMathEquals("2014-05-30T20:21||", "2014-05-30T20:21:00.000");
+    }
+
+    public void testMultipleAdjustments() {
+        assertDateMathEquals("2014-11-18||+1M-1M", "2014-11-18");
+        assertDateMathEquals("2014-11-18||+1M-1m", "2014-12-17T23:59");
+        assertDateMathEquals("2014-11-18||-1m+1M", "2014-12-17T23:59");
+        assertDateMathEquals("2014-11-18||+1M/M", "2014-12-01");
+        assertDateMathEquals("2014-11-18||+1M/M+1h", "2014-12-01T01");
+    }
+
+    public void testNow() {
+        final long now = parser.parse("2014-11-18T14:27:32", () -> 0, false, null);
+
+        assertDateMathEquals("now", "2014-11-18T14:27:32", now, false, null);
+        assertDateMathEquals("now+M", "2014-12-18T14:27:32", now, false, null);
+        assertDateMathEquals("now-2d", "2014-11-16T14:27:32", now, false, null);
+        assertDateMathEquals("now/m", "2014-11-18T14:27", now, false, null);
+
+        // timezone does not affect now
+        assertDateMathEquals("now/m", "2014-11-18T14:27", now, false, ZoneId.of("+02:00"));
+    }
+
+    public void testRoundingPreservesEpochAsBaseDate() {
+        // If a user only specifies times, then the date needs to always be 1970-01-01 regardless of rounding
+        CompoundDateTimeFormatter formatter = DateFormatters.forPattern("HH:mm:ss");
+        DateMathParser parser = new DateMathParser(formatter);
+        ZonedDateTime zonedDateTime = DateFormatters.toZonedDateTime(formatter.parse("04:52:20"));
+        assertThat(zonedDateTime.getYear(), is(1970));
+        long millisStart = zonedDateTime.toInstant().toEpochMilli();
+        assertEquals(millisStart, parser.parse("04:52:20", () -> 0, false, null));
+        // due to rounding up, we have to add the number of milliseconds here manually
+        long millisEnd = DateFormatters.toZonedDateTime(formatter.parse("04:52:20")).toInstant().toEpochMilli() + 999;
+        assertEquals(millisEnd, parser.parse("04:52:20", () -> 0, true, null));
+    }
+
+    // Implicit rounding happening when parts of the date are not specified
+    public void testImplicitRounding() {
+        assertDateMathEquals("2014-11-18", "2014-11-18", 0, false, null);
+        assertDateMathEquals("2014-11-18", "2014-11-18T23:59:59.999Z", 0, true, null);
+
+        assertDateMathEquals("2014-11-18T09:20", "2014-11-18T09:20", 0, false, null);
+        assertDateMathEquals("2014-11-18T09:20", "2014-11-18T09:20:59.999Z", 0, true, null);
+
+        assertDateMathEquals("2014-11-18", "2014-11-17T23:00:00.000Z", 0, false, ZoneId.of("CET"));
+        assertDateMathEquals("2014-11-18", "2014-11-18T22:59:59.999Z", 0, true, ZoneId.of("CET"));
+
+        assertDateMathEquals("2014-11-18T09:20", "2014-11-18T08:20:00.000Z", 0, false, ZoneId.of("CET"));
+        assertDateMathEquals("2014-11-18T09:20", "2014-11-18T08:20:59.999Z", 0, true, ZoneId.of("CET"));
+
+        // implicit rounding with explicit timezone in the date format
+        CompoundDateTimeFormatter formatter = DateFormatters.forPattern("yyyy-MM-ddXXX");
+        DateMathParser parser = new DateMathParser(formatter);
+        long time = parser.parse("2011-10-09+01:00", () -> 0, false, null);
+        assertEquals(this.parser.parse("2011-10-09T00:00:00.000+01:00", () -> 0), time);
+        time = parser.parse("2011-10-09+01:00", () -> 0, true, null);
+        assertEquals(this.parser.parse("2011-10-09T23:59:59.999+01:00", () -> 0), time);
+    }
+
+    // Explicit rounding using the || separator
+    public void testExplicitRounding() {
+        assertDateMathEquals("2014-11-18||/y", "2014-01-01", 0, false, null);
+        assertDateMathEquals("2014-11-18||/y", "2014-12-31T23:59:59.999", 0, true, null);
+        assertDateMathEquals("2014||/y", "2014-01-01", 0, false, null);
+        assertDateMathEquals("2014-01-01T00:00:00.001||/y", "2014-12-31T23:59:59.999", 0, true, null);
+        // rounding should also take into account time zone
+        assertDateMathEquals("2014-11-18||/y", "2013-12-31T23:00:00.000Z", 0, false, ZoneId.of("CET"));
+        assertDateMathEquals("2014-11-18||/y", "2014-12-31T22:59:59.999Z", 0, true, ZoneId.of("CET"));
+
+        assertDateMathEquals("2014-11-18||/M", "2014-11-01", 0, false, null);
+        assertDateMathEquals("2014-11-18||/M", "2014-11-30T23:59:59.999", 0, true, null);
+        assertDateMathEquals("2014-11||/M", "2014-11-01", 0, false, null);
+        assertDateMathEquals("2014-11||/M", "2014-11-30T23:59:59.999", 0, true, null);
+        assertDateMathEquals("2014-11-18||/M", "2014-10-31T23:00:00.000Z", 0, false, ZoneId.of("CET"));
+        assertDateMathEquals("2014-11-18||/M", "2014-11-30T22:59:59.999Z", 0, true, ZoneId.of("CET"));
+
+        assertDateMathEquals("2014-11-18T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-18T14||/w", "2014-11-23T23:59:59.999", 0, true, null);
+        assertDateMathEquals("2014-11-17T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-18T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-19T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-20T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-21T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-22T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-23T14||/w", "2014-11-17", 0, false, null);
+        assertDateMathEquals("2014-11-18||/w", "2014-11-23T23:59:59.999", 0, true, null);
+        assertDateMathEquals("2014-11-18||/w", "2014-11-16T23:00:00.000Z", 0, false, ZoneId.of("+01:00"));
+        assertDateMathEquals("2014-11-18||/w", "2014-11-17T01:00:00.000Z", 0, false, ZoneId.of("-01:00"));
+        assertDateMathEquals("2014-11-18||/w", "2014-11-16T23:00:00.000Z", 0, false, ZoneId.of("CET"));
+        assertDateMathEquals("2014-11-18||/w", "2014-11-23T22:59:59.999Z", 0, true, ZoneId.of("CET"));
+        assertDateMathEquals("2014-07-22||/w", "2014-07-20T22:00:00.000Z", 0, false, ZoneId.of("CET")); // with DST
+
+        assertDateMathEquals("2014-11-18T14||/d", "2014-11-18", 0, false, null);
+        assertDateMathEquals("2014-11-18T14||/d", "2014-11-18T23:59:59.999", 0, true, null);
+        assertDateMathEquals("2014-11-18||/d", "2014-11-18", 0, false, null);
+        assertDateMathEquals("2014-11-18||/d", "2014-11-18T23:59:59.999", 0, true, null);
+
+        assertDateMathEquals("2014-11-18T14:27||/h", "2014-11-18T14", 0, false, null);
+        assertDateMathEquals("2014-11-18T14:27||/h", "2014-11-18T14:59:59.999", 0, true, null);
+        assertDateMathEquals("2014-11-18T14||/H", "2014-11-18T14", 0, false, null);
+        assertDateMathEquals("2014-11-18T14||/H", "2014-11-18T14:59:59.999", 0, true, null);
+        assertDateMathEquals("2014-11-18T14:27||/h", "2014-11-18T14", 0, false, null);
+        assertDateMathEquals("2014-11-18T14:27||/h", "2014-11-18T14:59:59.999", 0, true, null);
+        assertDateMathEquals("2014-11-18T14||/H", "2014-11-18T14", 0, false, null);
+        assertDateMathEquals("2014-11-18T14||/H", "2014-11-18T14:59:59.999", 0, true, null);
+
+        assertDateMathEquals("2014-11-18T14:27:32||/m", "2014-11-18T14:27", 0, false, null);
+        assertDateMathEquals("2014-11-18T14:27:32||/m", "2014-11-18T14:27:59.999", 0, true, null);
+        assertDateMathEquals("2014-11-18T14:27||/m", "2014-11-18T14:27", 0, false, null);
+        assertDateMathEquals("2014-11-18T14:27||/m", "2014-11-18T14:27:59.999", 0, true, null);
+
+        assertDateMathEquals("2014-11-18T14:27:32.123||/s", "2014-11-18T14:27:32", 0, false, null);
+        assertDateMathEquals("2014-11-18T14:27:32.123||/s", "2014-11-18T14:27:32.999", 0, true, null);
+        assertDateMathEquals("2014-11-18T14:27:32||/s", "2014-11-18T14:27:32", 0, false, null);
+        assertDateMathEquals("2014-11-18T14:27:32||/s", "2014-11-18T14:27:32.999", 0, true, null);
+    }
+
+    public void testTimestamps() {
+        assertDateMathEquals("1418248078000", "2014-12-10T21:47:58.000");
+        assertDateMathEquals("32484216259000", "2999-05-20T17:24:19.000");
+        assertDateMathEquals("253382837059000", "9999-05-20T17:24:19.000");
+
+        // datemath still works on timestamps
+        assertDateMathEquals("1418248078000||/m", "2014-12-10T21:47:00.000");
+
+        // also check other time units
+        DateMathParser parser = new DateMathParser(DateFormatters.forPattern("epoch_second||dateOptionalTime"));
+        long datetime = parser.parse("1418248078", () -> 0);
+        assertDateEquals(datetime, "1418248078", "2014-12-10T21:47:58.000");
+
+        // a timestamp before 10000 is a year
+        assertDateMathEquals("9999", "9999-01-01T00:00:00.000");
+        // 10000 is also a year, breaking bwc, used to be a timestamp
+        assertDateMathEquals("10000", "10000-01-01T00:00:00.000");
+        // but 10000 with T is still a date format
+        assertDateMathEquals("10000T", "10000-01-01T00:00:00.000");
+    }
+
+    void assertParseException(String msg, String date, String exc) {
+        try {
+            parser.parse(date, () -> 0);
+            fail("Date: " + date + "\n" + msg);
+        } catch (ElasticsearchParseException e) {
+            assertThat(ExceptionsHelper.detailedMessage(e), containsString(exc));
+        }
+    }
+
+    public void testIllegalMathFormat() {
+        assertParseException("Expected date math unsupported operator exception", "2014-11-18||*5", "operator not supported");
+        assertParseException("Expected date math incompatible rounding exception", "2014-11-18||/2m", "rounding");
+        assertParseException("Expected date math illegal unit type exception", "2014-11-18||+2a", "unit [a] not supported");
+        assertParseException("Expected date math truncation exception", "2014-11-18||+12", "truncated");
+        assertParseException("Expected date math truncation exception", "2014-11-18||-", "truncated");
+    }
+
+    public void testIllegalDateFormat() {
+        assertParseException("Expected bad timestamp exception", Long.toString(Long.MAX_VALUE) + "0", "failed to parse date field");
+        assertParseException("Expected bad date format exception", "123bogus", "could not be parsed");
+    }
+
+    public void testOnlyCallsNowIfNecessary() {
+        final AtomicBoolean called = new AtomicBoolean();
+        final LongSupplier now = () -> {
+            called.set(true);
+            return 42L;
+        };
+        parser.parse("2014-11-18T14:27:32", now, false, null);
+        assertFalse(called.get());
+        parser.parse("now/d", now, false, null);
+        assertTrue(called.get());
+    }
+
+    private void assertDateMathEquals(String toTest, String expected) {
+        assertDateMathEquals(toTest, expected, 0, false, null);
+    }
+
+    private void assertDateMathEquals(String toTest, String expected, final long now, boolean roundUp, ZoneId timeZone) {
+        long gotMillis = parser.parse(toTest, () -> now, roundUp, timeZone);
+        assertDateEquals(gotMillis, toTest, expected);
+    }
+
+    private void assertDateEquals(long gotMillis, String original, String expected) {
+        long expectedMillis = parser.parse(expected, () -> 0);
+        if (gotMillis != expectedMillis) {
+            ZonedDateTime zonedDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(gotMillis), ZoneOffset.UTC);
+            fail("Date math not equal\n" +
+                "Original              : " + original + "\n" +
+                "Parsed                : " + formatter.format(zonedDateTime) + "\n" +
+                "Expected              : " + expected + "\n" +
+                "Expected milliseconds : " + expectedMillis + "\n" +
+                "Actual milliseconds   : " + gotMillis + "\n");
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java
index a893fb63ec8cc..1a0d0dead6e6d 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java
@@ -28,6 +28,7 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.util.Arrays;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -54,8 +55,21 @@ private void testGuessType(XContentType type) throws IOException {
         builder.field("field1", "value1");
         builder.endObject();
 
-        assertThat(XContentHelper.xContentType(BytesReference.bytes(builder)), equalTo(type));
-        assertThat(XContentFactory.xContentType(BytesReference.bytes(builder).streamInput()), equalTo(type));
+        final BytesReference bytes;
+        if (type == XContentType.JSON && randomBoolean()) {
+            final int length = randomIntBetween(0, 8 * XContentFactory.GUESS_HEADER_LENGTH);
+            final String content = Strings.toString(builder);
+            final StringBuilder sb = new StringBuilder(length + content.length());
+            final char[] chars = new char[length];
+            Arrays.fill(chars, ' ');
+            sb.append(new String(chars)).append(content);
+            bytes = new BytesArray(sb.toString());
+        } else {
+            bytes = BytesReference.bytes(builder);
+        }
+
+        assertThat(XContentHelper.xContentType(bytes), equalTo(type));
+        assertThat(XContentFactory.xContentType(bytes.streamInput()), equalTo(type));
 
         // CBOR is binary, cannot use String
         if (type != XContentType.CBOR && type != XContentType.SMILE) {
diff --git a/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java
index f688291237813..4d190921f2da9 100644
--- a/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java
+++ b/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java
@@ -37,13 +37,9 @@
 import org.elasticsearch.test.disruption.SlowClusterStateProcessing;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -175,24 +171,15 @@ public void testClusterJoinDespiteOfPublishingIssues() throws Exception {
 
         logger.info("allowing requests from non master [{}] to master [{}], waiting for two join request", nonMasterNode, masterNode);
         final CountDownLatch countDownLatch = new CountDownLatch(2);
-        nonMasterTransportService.addDelegate(masterTranspotService, new MockTransportService.DelegateTransport(nonMasterTransportService
-                .original()) {
-            @Override
-            protected void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request,
-                                       TransportRequestOptions options) throws IOException {
-                if (action.equals(MembershipAction.DISCOVERY_JOIN_ACTION_NAME)) {
-                    countDownLatch.countDown();
-                }
-                super.sendRequest(connection, requestId, action, request, options);
+        nonMasterTransportService.addSendBehavior(masterTransportService, (connection, requestId, action, request, options) -> {
+            if (action.equals(MembershipAction.DISCOVERY_JOIN_ACTION_NAME)) {
+                countDownLatch.countDown();
             }
-
-            @Override
-            public Transport.Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
-                return super.openConnection(node, profile);
-            }
-
+            connection.sendRequest(requestId, action, request, options);
         });
 
+        nonMasterTransportService.addConnectBehavior(masterTransportService, Transport::openConnection);
+
         countDownLatch.await();
 
         logger.info("waiting for cluster to reform");
diff --git a/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryTests.java b/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryTests.java
index d045adcaead21..57f7bfec6ef38 100644
--- a/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryTests.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -64,11 +63,6 @@ public void setInitialState(ClusterState initialState) {
                                 clusterState.set(initialState);
                             }
 
-                            @Override
-                            public ClusterState.Builder newClusterStateBuilder() {
-                                return ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings));
-                            }
-
                             @Override
                             public void onNewClusterState(String source, Supplier<ClusterState> clusterStateSupplier,
                                                           ClusterApplyListener listener) {
diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
index eef926a1e1238..4ab738f5c7bc3 100644
--- a/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
@@ -21,7 +21,6 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.Constants;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -29,7 +28,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNode.Role;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -42,14 +40,13 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.MockTcpTransport;
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transport;
@@ -152,14 +149,7 @@ public void testSimplePings() throws IOException, InterruptedException, Executio
             new NoneCircuitBreakerService(),
             new NamedWriteableRegistry(Collections.emptyList()),
             networkService,
-            v) {
-            @Override
-            public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
-                                      CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
-                throws ConnectTransportException {
-                throw new AssertionError("zen pings should never connect to node (got [" + node + "])");
-            }
-        };
+            v);
 
         NetworkHandle handleA = startServices(settings, threadPool, "UZP_A", Version.CURRENT, supplier);
         closeables.push(handleA.transportService);
diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
index 6dbf80d9be675..2c4fb0c7e8db2 100644
--- a/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
@@ -306,11 +306,6 @@ public void setInitialState(ClusterState initialState) {
 
             }
 
-            @Override
-            public ClusterState.Builder newClusterStateBuilder() {
-                return ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings));
-            }
-
             @Override
             public void onNewClusterState(String source, Supplier<ClusterState> clusterStateSupplier, ClusterApplyListener listener) {
                 listener.onSuccess(source);
diff --git a/server/src/test/java/org/elasticsearch/document/ShardInfoIT.java b/server/src/test/java/org/elasticsearch/document/ShardInfoIT.java
index 5a5f279985f4c..682b1deb14662 100644
--- a/server/src/test/java/org/elasticsearch/document/ShardInfoIT.java
+++ b/server/src/test/java/org/elasticsearch/document/ShardInfoIT.java
@@ -91,6 +91,7 @@ public void testBulkWithUpdateItems() throws Exception {
 
         BulkResponse bulkResponse = bulkRequestBuilder.get();
         for (BulkItemResponse item : bulkResponse) {
+            assertThat(item.getFailure(), nullValue());
             assertThat(item.isFailed(), equalTo(false));
             assertShardInfo(item.getResponse());
         }
diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
index bd3d5beeaf5c0..3e4a3dce09153 100644
--- a/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
@@ -27,14 +27,13 @@
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
-import java.util.Collections;
 
 public class GatewayServiceTests extends ESTestCase {
 
     private GatewayService createService(Settings.Builder settings) {
         ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "GatewayServiceTests").build(),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                null, Collections.emptyMap());
+                null);
         return new GatewayService(settings.build(),
                 null, clusterService, null, null, null, null);
     }
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index f47cd11e1b4ec..9151fa24fc9a4 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -1978,7 +1978,7 @@ private static class MockAppender extends AbstractAppender {
         @Override
         public void append(LogEvent event) {
             final String formattedMessage = event.getMessage().getFormattedMessage();
-            if (event.getLevel() == Level.TRACE && event.getMarker().getName().contains("[index][0] ")) {
+            if (event.getLevel() == Level.TRACE && event.getMarker().getName().contains("[index][0]")) {
                 if (event.getLoggerName().endsWith(".IW") &&
                     formattedMessage.contains("IW: now apply all deletes")) {
                     sawIndexWriterMessage = true;
@@ -2024,7 +2024,6 @@ public void testIndexWriterInfoStream() throws IllegalAccessException, IOExcepti
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32430")
     public void testSeqNoAndCheckpoints() throws IOException {
         final int opCount = randomIntBetween(1, 256);
         long primarySeqNo = SequenceNumbers.NO_OPS_PERFORMED;
@@ -2034,10 +2033,12 @@ public void testSeqNoAndCheckpoints() throws IOException {
         long replicaLocalCheckpoint = SequenceNumbers.NO_OPS_PERFORMED;
         final long globalCheckpoint;
         long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
+        IOUtils.close(store, engine);
+        store = createStore();
         InternalEngine initialEngine = null;
 
         try {
-            initialEngine = engine;
+            initialEngine = createEngine(defaultSettings, store, createTempDir(), newLogMergePolicy(), null);
             final ShardRouting primary = TestShardRouting.newShardRouting("test", shardId.id(), "node1", null, true,
                 ShardRoutingState.STARTED, allocationId);
             final ShardRouting replica = TestShardRouting.newShardRouting(shardId, "node2", false, ShardRoutingState.STARTED);
@@ -2053,7 +2054,7 @@ public void testSeqNoAndCheckpoints() throws IOException {
                     // we have some docs indexed, so delete one of them
                     id = randomFrom(indexedIds);
                     final Engine.Delete delete = new Engine.Delete(
-                        "test", id, newUid(id), SequenceNumbers.UNASSIGNED_SEQ_NO, 0,
+                        "test", id, newUid(id), SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm.get(),
                         rarely() ? 100 : Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, 0);
                     final Engine.DeleteResult result = initialEngine.delete(delete);
                     if (result.getResultType() == Engine.Result.Type.SUCCESS) {
@@ -2070,7 +2071,7 @@ public void testSeqNoAndCheckpoints() throws IOException {
                     id = randomFrom(ids);
                     ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null);
                     final Engine.Index index = new Engine.Index(newUid(doc), doc,
-                        SequenceNumbers.UNASSIGNED_SEQ_NO, 0,
+                        SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm.get(),
                         rarely() ? 100 : Versions.MATCH_ANY, VersionType.INTERNAL,
                         PRIMARY, 0, -1, false);
                     final Engine.IndexResult result = initialEngine.index(index);
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java
index 43b9a01560ca8..f1ffe01bfc5e0 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java
@@ -23,29 +23,75 @@
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
-import org.joda.time.ReadableDateTime;
 
 import java.io.IOException;
+import java.security.AccessControlContext;
+import java.security.AccessController;
+import java.security.PermissionCollection;
+import java.security.Permissions;
+import java.security.PrivilegedAction;
+import java.security.ProtectionDomain;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Function;
 
+import static org.hamcrest.Matchers.containsInAnyOrder;
 
 public class ScriptDocValuesDatesTests extends ESTestCase {
-    public void test() throws IOException {
+
+    public void testJavaTime() throws IOException {
+        assertDateDocValues(true);
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32779")
+    public void testJodaTimeBwc() throws IOException {
+        assertDateDocValues(false, "The joda time api for doc values is deprecated." +
+            " Use -Des.scripting.use_java_time=true to use the java time api for date field doc values");
+    }
+
+    public void assertDateDocValues(boolean useJavaTime, String... expectedWarnings) throws IOException {
+        final Function<Long, Object> datetimeCtor;
+        if (useJavaTime) {
+            datetimeCtor = millis -> ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneOffset.UTC);
+        } else {
+            datetimeCtor = millis -> new DateTime(millis, DateTimeZone.UTC);
+        }
         long[][] values = new long[between(3, 10)][];
-        ReadableDateTime[][] expectedDates = new ReadableDateTime[values.length][];
+        Object[][] expectedDates = new Object[values.length][];
         for (int d = 0; d < values.length; d++) {
             values[d] = new long[randomBoolean() ? randomBoolean() ? 0 : 1 : between(2, 100)];
-            expectedDates[d] = new ReadableDateTime[values[d].length];
+            expectedDates[d] = new Object[values[d].length];
             for (int i = 0; i < values[d].length; i++) {
-                expectedDates[d][i] = new DateTime(randomNonNegativeLong(), DateTimeZone.UTC);
-                values[d][i] = expectedDates[d][i].getMillis();
+                values[d][i] = randomNonNegativeLong();
+                expectedDates[d][i] = datetimeCtor.apply(values[d][i]);
             }
         }
-        Dates dates = wrap(values);
+
+        Set<String> warnings = new HashSet<>();
+        Dates dates = wrap(values, deprecationMessage -> {
+            warnings.add(deprecationMessage);
+            /* Create a temporary directory to prove we are running with the
+             * server's permissions. */
+            createTempDir();
+        }, useJavaTime);
+        // each call to get or getValue will be run with limited permissions, just as they are in scripts
+        PermissionCollection noPermissions = new Permissions();
+        AccessControlContext noPermissionsAcc = new AccessControlContext(
+            new ProtectionDomain[] {
+                new ProtectionDomain(null, noPermissions)
+            }
+        );
+
         for (int round = 0; round < 10; round++) {
             int d = between(0, values.length - 1);
             dates.setNextDocId(d);
             if (expectedDates[d].length > 0) {
-                assertEquals(expectedDates[d][0] , dates.getValue());
+                Object dateValue = AccessController.doPrivileged((PrivilegedAction<Object>) dates::getValue, noPermissionsAcc);
+                assertEquals(expectedDates[d][0] , dateValue);
             } else {
                 Exception e = expectThrows(IllegalStateException.class, () -> dates.getValue());
                 assertEquals("A document doesn't have a value for a field! " +
@@ -54,15 +100,16 @@ public void test() throws IOException {
 
             assertEquals(values[d].length, dates.size());
             for (int i = 0; i < values[d].length; i++) {
-                assertEquals(expectedDates[d][i], dates.get(i));
+                final int ndx = i;
+                Object dateValue = AccessController.doPrivileged((PrivilegedAction<Object>) () -> dates.get(ndx), noPermissionsAcc);
+                assertEquals(expectedDates[d][i], dateValue);
             }
-
-            Exception e = expectThrows(UnsupportedOperationException.class, () -> dates.add(new DateTime()));
-            assertEquals("doc values are unmodifiable", e.getMessage());
         }
+
+        assertThat(warnings, containsInAnyOrder(expectedWarnings));
     }
 
-    private Dates wrap(long[][] values) {
+    private Dates wrap(long[][] values, Consumer<String> deprecationHandler, boolean useJavaTime) {
         return new Dates(new AbstractSortedNumericDocValues() {
             long[] current;
             int i;
@@ -81,6 +128,6 @@ public int docValueCount() {
             public long nextValue() {
                 return current[i++];
             }
-        });
+        }, deprecationHandler, useJavaTime);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/get/DocumentFieldTests.java b/server/src/test/java/org/elasticsearch/index/get/DocumentFieldTests.java
index 830e41996e9b3..e2e3d4df67cf7 100644
--- a/server/src/test/java/org/elasticsearch/index/get/DocumentFieldTests.java
+++ b/server/src/test/java/org/elasticsearch/index/get/DocumentFieldTests.java
@@ -26,7 +26,11 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.mapper.RoutingFieldMapper;
+import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.IgnoredFieldMapper;
+import org.elasticsearch.index.mapper.IndexFieldMapper;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.TypeFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.RandomObjects;
 
@@ -98,14 +102,28 @@ private static DocumentField mutateDocumentField(DocumentField documentField) {
 
     public static Tuple<DocumentField, DocumentField> randomDocumentField(XContentType xContentType) {
         if (randomBoolean()) {
-            String fieldName = randomFrom(RoutingFieldMapper.NAME);
-            DocumentField documentField = new DocumentField(fieldName, Collections.singletonList(randomAlphaOfLengthBetween(3, 10)));
+            String metaField = randomValueOtherThanMany(field -> field.equals(TypeFieldMapper.NAME)
+                    || field.equals(IndexFieldMapper.NAME) || field.equals(IdFieldMapper.NAME),
+                () -> randomFrom(MapperService.getAllMetaFields()));
+            DocumentField documentField;
+            if (metaField.equals(IgnoredFieldMapper.NAME)) {
+                int numValues = randomIntBetween(1, 3);
+                List<Object> ignoredFields = new ArrayList<>(numValues);
+                for (int i = 0; i < numValues; i++) {
+                    ignoredFields.add(randomAlphaOfLengthBetween(3, 10));
+                }
+                documentField = new DocumentField(metaField, ignoredFields);
+            } else {
+                //meta fields are single value only, besides _ignored
+                documentField = new DocumentField(metaField, Collections.singletonList(randomAlphaOfLengthBetween(3, 10)));
+            }
             return Tuple.tuple(documentField, documentField);
+        } else {
+            String fieldName = randomAlphaOfLengthBetween(3, 10);
+            Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xContentType);
+            DocumentField input = new DocumentField(fieldName, tuple.v1());
+            DocumentField expected = new DocumentField(fieldName, tuple.v2());
+            return Tuple.tuple(input, expected);
         }
-        String fieldName = randomAlphaOfLengthBetween(3, 10);
-        Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xContentType);
-        DocumentField input = new DocumentField(fieldName, tuple.v1());
-        DocumentField expected = new DocumentField(fieldName, tuple.v2());
-        return Tuple.tuple(input, expected);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java b/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java
index a38d183299cdd..1cc2612041f46 100644
--- a/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java
+++ b/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java
@@ -90,7 +90,6 @@ public void testToAndFromXContentEmbedded() throws Exception {
         XContentType xContentType = randomFrom(XContentType.values());
         Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
         GetResult getResult = tuple.v1();
-
         // We don't expect to retrieve the index/type/id of the GetResult because they are not rendered
         // by the toXContentEmbedded method.
         GetResult expectedGetResult = new GetResult(null, null, null, -1,
@@ -106,7 +105,6 @@ public void testToAndFromXContentEmbedded() throws Exception {
             parsedEmbeddedGetResult = GetResult.fromXContentEmbedded(parser);
             assertNull(parser.nextToken());
         }
-
         assertEquals(expectedGetResult, parsedEmbeddedGetResult);
         //print the parsed object out and test that the output is the same as the original output
         BytesReference finalBytes = toXContentEmbedded(parsedEmbeddedGetResult, xContentType, humanReadable);
@@ -203,16 +201,17 @@ public static Tuple<GetResult, GetResult> randomGetResult(XContentType xContentT
         return Tuple.tuple(getResult, expectedGetResult);
     }
 
-    private static Tuple<Map<String, DocumentField>,Map<String, DocumentField>> randomDocumentFields(XContentType xContentType) {
+    public static Tuple<Map<String, DocumentField>,Map<String, DocumentField>> randomDocumentFields(XContentType xContentType) {
         int numFields = randomIntBetween(2, 10);
         Map<String, DocumentField> fields = new HashMap<>(numFields);
         Map<String, DocumentField> expectedFields = new HashMap<>(numFields);
-        for (int i = 0; i < numFields; i++) {
+        while (fields.size() < numFields) {
             Tuple<DocumentField, DocumentField> tuple = randomDocumentField(xContentType);
             DocumentField getField = tuple.v1();
             DocumentField expectedGetField = tuple.v2();
-            fields.put(getField.getName(), getField);
-            expectedFields.put(expectedGetField.getName(), expectedGetField);
+            if (fields.putIfAbsent(getField.getName(), getField) == null) {
+                assertNull(expectedFields.putIfAbsent(expectedGetField.getName(), expectedGetField));
+            }
         }
         return Tuple.tuple(fields, expectedFields);
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
index 8638e16e29af2..fd98a8199a22f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
@@ -142,7 +142,7 @@ public void testParsesBooleansStrict() throws IOException {
                 .endObject());
         MapperParsingException ex = expectThrows(MapperParsingException.class,
                 () -> defaultMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON)));
-        assertEquals("failed to parse [field]", ex.getMessage());
+        assertEquals("failed to parse field [field] of type [boolean]", ex.getMessage());
     }
 
     public void testMultiFields() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 098aacc799f5d..76ca6aa7ea8d4 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -125,6 +125,35 @@ public void testDotsWithExistingNestedMapper() throws Exception {
                 e.getMessage());
     }
 
+    public void testUnexpectedFieldMappingType() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
+                .startObject("foo").field("type", "long").endObject()
+                .startObject("bar").field("type", "boolean").endObject()
+                .startObject("geo").field("type", "geo_shape").endObject()
+                .endObject().endObject().endObject());
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+        {
+            BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("foo", true).endObject());
+            MapperException exception = expectThrows(MapperException.class,
+                    () -> mapper.parse(SourceToParse.source("test", "type", "1", bytes, XContentType.JSON)));
+            assertThat(exception.getMessage(), containsString("failed to parse field [foo] of type [long]"));
+        }
+        {
+            BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("bar", "bar").endObject());
+            MapperException exception = expectThrows(MapperException.class,
+                    () -> mapper.parse(SourceToParse.source("test", "type", "2", bytes, XContentType.JSON)));
+            assertThat(exception.getMessage(), containsString("failed to parse field [bar] of type [boolean]"));
+        }
+        {
+            BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("geo", 123).endObject());
+            MapperException exception = expectThrows(MapperException.class,
+                    () -> mapper.parse(SourceToParse.source("test", "type", "2", bytes, XContentType.JSON)));
+            assertThat(exception.getMessage(), containsString("failed to parse field [geo] of type [geo_shape]"));
+        }
+
+    }
+
     public void testDotsWithDynamicNestedMapper() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
index 39cc3c8219ceb..b27b9e26ab7dc 100644
--- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
@@ -22,8 +22,10 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
@@ -37,6 +39,7 @@
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
+import java.util.Collections;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -49,24 +52,7 @@
 public class QueryShardContextTests extends ESTestCase {
 
     public void testFailIfFieldMappingNotFound() {
-        IndexMetaData.Builder indexMetadataBuilder = new IndexMetaData.Builder("index");
-        indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT)
-            .put("index.number_of_shards", 1)
-            .put("index.number_of_replicas", 1)
-        );
-        IndexMetaData indexMetaData = indexMetadataBuilder.build();
-        IndexSettings indexSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
-        MapperService mapperService = mock(MapperService.class);
-        when(mapperService.getIndexSettings()).thenReturn(indexSettings);
-        when(mapperService.index()).thenReturn(indexMetaData.getIndex());
-        final long nowInMillis = randomNonNegativeLong();
-
-        QueryShardContext context = new QueryShardContext(
-            0, indexSettings, null, (mappedFieldType, idxName) ->
-                mappedFieldType.fielddataBuilder(idxName).build(indexSettings, mappedFieldType, null, null, null)
-                , mapperService, null, null, xContentRegistry(), writableRegistry(), null, null,
-            () -> nowInMillis, null);
-
+        QueryShardContext context = createQueryShardContext(IndexMetaData.INDEX_UUID_NA_VALUE, null);
         context.setAllowUnmappedFields(false);
         MappedFieldType fieldType = new TextFieldMapper.TextFieldType();
         MappedFieldType result = context.failIfFieldMappingNotFound("name", fieldType);
@@ -91,30 +77,16 @@ public void testFailIfFieldMappingNotFound() {
     }
 
     public void testClusterAlias() throws IOException {
-        IndexMetaData.Builder indexMetadataBuilder = new IndexMetaData.Builder("index");
-        indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT)
-            .put("index.number_of_shards", 1)
-            .put("index.number_of_replicas", 1)
-        );
-        IndexMetaData indexMetaData = indexMetadataBuilder.build();
-        IndexSettings indexSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
-        MapperService mapperService = mock(MapperService.class);
-        when(mapperService.getIndexSettings()).thenReturn(indexSettings);
-        when(mapperService.index()).thenReturn(indexMetaData.getIndex());
-        final long nowInMillis = randomNonNegativeLong();
+        final String clusterAlias = randomBoolean() ? null : "remote_cluster";
+        QueryShardContext context = createQueryShardContext(IndexMetaData.INDEX_UUID_NA_VALUE, clusterAlias);
 
-        Mapper.BuilderContext ctx = new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath());
+
+        Mapper.BuilderContext ctx = new Mapper.BuilderContext(context.getIndexSettings().getSettings(), new ContentPath());
         IndexFieldMapper mapper = new IndexFieldMapper.Builder(null).build(ctx);
-        final String clusterAlias = randomBoolean() ? null : "remote_cluster";
-        QueryShardContext context = new QueryShardContext(
-            0, indexSettings, null, (mappedFieldType, indexname) ->
-            mappedFieldType.fielddataBuilder(indexname).build(indexSettings, mappedFieldType, null, null, mapperService)
-            , mapperService, null, null, xContentRegistry(), writableRegistry(), null, null,
-            () -> nowInMillis, clusterAlias);
 
         IndexFieldData<?> forField = context.getForField(mapper.fieldType());
-        String expected = clusterAlias == null ? indexMetaData.getIndex().getName()
-            : clusterAlias + ":" + indexMetaData.getIndex().getName();
+        String expected = clusterAlias == null ? context.getIndexSettings().getIndexMetaData().getIndex().getName()
+            : clusterAlias + ":" + context.getIndexSettings().getIndex().getName();
         assertEquals(expected, ((AbstractAtomicOrdinalsFieldData)forField.load(null)).getOrdinalsValues().lookupOrd(0).utf8ToString());
         Query query = mapper.fieldType().termQuery("index", context);
         if (clusterAlias == null) {
@@ -133,4 +105,32 @@ public void testClusterAlias() throws IOException {
         assertThat(query, Matchers.instanceOf(MatchNoDocsQuery.class));
     }
 
+    public void testGetFullyQualifiedIndex() {
+        String clusterAlias = randomAlphaOfLengthBetween(5, 10);
+        String indexUuid = randomAlphaOfLengthBetween(3, 10);
+        QueryShardContext shardContext = createQueryShardContext(indexUuid, clusterAlias);
+        assertThat(shardContext.getFullyQualifiedIndex().getName(), equalTo(clusterAlias + ":index"));
+        assertThat(shardContext.getFullyQualifiedIndex().getUUID(), equalTo(indexUuid));
+    }
+
+    public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) {
+        IndexMetaData.Builder indexMetadataBuilder = new IndexMetaData.Builder("index");
+        indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT)
+            .put("index.number_of_shards", 1)
+            .put("index.number_of_replicas", 1)
+            .put(IndexMetaData.SETTING_INDEX_UUID, indexUuid)
+        );
+        IndexMetaData indexMetaData = indexMetadataBuilder.build();
+        IndexSettings indexSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
+        MapperService mapperService = mock(MapperService.class);
+        when(mapperService.getIndexSettings()).thenReturn(indexSettings);
+        when(mapperService.index()).thenReturn(indexMetaData.getIndex());
+        final long nowInMillis = randomNonNegativeLong();
+
+        return new QueryShardContext(
+            0, indexSettings, null, (mappedFieldType, idxName) ->
+            mappedFieldType.fielddataBuilder(idxName).build(indexSettings, mappedFieldType, null, null, null)
+            , mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), null, null,
+            () -> nowInMillis, clusterAlias);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardExceptionTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardExceptionTests.java
new file mode 100644
index 0000000000000..8557f568e078f
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardExceptionTests.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.index.Index;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class QueryShardExceptionTests extends ESTestCase {
+
+    public void testCreateFromQueryShardContext() {
+        String indexUuid = randomAlphaOfLengthBetween(5, 10);
+        String clusterAlias = randomAlphaOfLengthBetween(5, 10);
+        QueryShardContext queryShardContext = QueryShardContextTests.createQueryShardContext(indexUuid, clusterAlias);
+        {
+            QueryShardException queryShardException = new QueryShardException(queryShardContext, "error");
+            assertThat(queryShardException.getIndex().getName(), equalTo(clusterAlias + ":index"));
+            assertThat(queryShardException.getIndex().getUUID(), equalTo(indexUuid));
+        }
+        {
+            QueryShardException queryShardException = new QueryShardException(queryShardContext, "error", new IllegalArgumentException());
+            assertThat(queryShardException.getIndex().getName(), equalTo(clusterAlias + ":index"));
+            assertThat(queryShardException.getIndex().getUUID(), equalTo(indexUuid));
+        }
+    }
+
+    public void testCreateFromIndex() {
+        String indexUuid = randomAlphaOfLengthBetween(5, 10);
+        String indexName = randomAlphaOfLengthBetween(5, 10);
+        Index index = new Index(indexName, indexUuid);
+        QueryShardException queryShardException = new QueryShardException(index, "error", new IllegalArgumentException());
+        assertThat(queryShardException.getIndex().getName(), equalTo(indexName));
+        assertThat(queryShardException.getIndex().getUUID(), equalTo(indexUuid));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
index a2d6e3ab361b7..6cde10308c6e7 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
@@ -233,7 +233,7 @@ public void testFieldsCannotBeSetToNull() {
 
     public void testDefaultFieldParsing() throws IOException {
         assumeTrue("5.x behaves differently, so skip on non-6.x indices",
-                indexVersionCreated.onOrAfter(Version.V_6_0_0_alpha1));
+                indexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1));
 
         String query = randomAlphaOfLengthBetween(1, 10).toLowerCase(Locale.ROOT);
         String contentString = "{\n" +
diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
index 54f8536563d9c..73aa1667f30f4 100644
--- a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
@@ -32,7 +32,6 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.elasticsearch.test.AbstractBuilderTestCase.STRING_ALIAS_FIELD_NAME;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -143,7 +142,7 @@ public void testWithMetaDataField() throws IOException {
 
     public void testIndexWildcard() throws IOException {
         QueryShardContext context = createShardContext();
-        String index = context.getFullyQualifiedIndexName();
+        String index = context.getFullyQualifiedIndex().getName();
 
         Query query = new WildcardQueryBuilder("_index", index).doToQuery(context);
         assertThat(query instanceof MatchAllDocsQuery, equalTo(true));
diff --git a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
index b05b1e5cc5ca0..f38550d70413b 100644
--- a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
@@ -59,12 +59,15 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.index.translog.SnapshotMatchers.containsOperationsInAnyOrder;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
@@ -221,7 +224,7 @@ public void testConflictingOpsOnReplica() throws Exception {
             }
 
             logger.info("--> promoting replica to primary " + replica1.routingEntry());
-            shards.promoteReplicaToPrimary(replica1);
+            shards.promoteReplicaToPrimary(replica1).get();
             indexRequest = new IndexRequest(index.getName(), "type", "1").source("{ \"f\": \"2\"}", XContentType.JSON);
             shards.index(indexRequest);
             shards.refresh("test");
@@ -234,6 +237,102 @@ public void testConflictingOpsOnReplica() throws Exception {
         }
     }
 
+    public void testReplicaTermIncrementWithConcurrentPrimaryPromotion() throws Exception {
+        Map<String, String> mappings =
+            Collections.singletonMap("type", "{ \"type\": { \"properties\": { \"f\": { \"type\": \"keyword\"} }}}");
+        try (ReplicationGroup shards = new ReplicationGroup(buildIndexMetaData(2, mappings))) {
+            shards.startAll();
+            long primaryPrimaryTerm = shards.getPrimary().getPendingPrimaryTerm();
+            List<IndexShard> replicas = shards.getReplicas();
+            IndexShard replica1 = replicas.get(0);
+            IndexShard replica2 = replicas.get(1);
+
+            shards.promoteReplicaToPrimary(replica1, (shard, listener) -> {});
+            long newReplica1Term = replica1.getPendingPrimaryTerm();
+            assertEquals(primaryPrimaryTerm + 1, newReplica1Term);
+
+            assertEquals(primaryPrimaryTerm, replica2.getPendingPrimaryTerm());
+
+            IndexRequest indexRequest = new IndexRequest(index.getName(), "type", "1").source("{ \"f\": \"1\"}", XContentType.JSON);
+            BulkShardRequest replicationRequest = indexOnPrimary(indexRequest, replica1);
+
+            CyclicBarrier barrier = new CyclicBarrier(2);
+            Thread t1 = new Thread(() -> {
+                try {
+                    barrier.await();
+                    indexOnReplica(replicationRequest, shards, replica2, newReplica1Term);
+                } catch (IllegalStateException ise) {
+                    assertThat(ise.getMessage(), either(containsString("is too old")).or(containsString("cannot be a replication target")));
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            });
+            Thread t2 = new Thread(() -> {
+                try {
+                    barrier.await();
+                    shards.promoteReplicaToPrimary(replica2).get();
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            });
+            t2.start();
+            t1.start();
+            t1.join();
+            t2.join();
+
+            assertEquals(newReplica1Term + 1, replica2.getPendingPrimaryTerm());
+        }
+    }
+
+    public void testReplicaOperationWithConcurrentPrimaryPromotion() throws Exception {
+        Map<String, String> mappings =
+            Collections.singletonMap("type", "{ \"type\": { \"properties\": { \"f\": { \"type\": \"keyword\"} }}}");
+        try (ReplicationGroup shards = new ReplicationGroup(buildIndexMetaData(1, mappings))) {
+            shards.startAll();
+            long primaryPrimaryTerm = shards.getPrimary().getPendingPrimaryTerm();
+            IndexRequest indexRequest = new IndexRequest(index.getName(), "type", "1").source("{ \"f\": \"1\"}", XContentType.JSON);
+            BulkShardRequest replicationRequest = indexOnPrimary(indexRequest, shards.getPrimary());
+
+            List<IndexShard> replicas = shards.getReplicas();
+            IndexShard replica = replicas.get(0);
+
+            CyclicBarrier barrier = new CyclicBarrier(2);
+            AtomicBoolean successFullyIndexed = new AtomicBoolean();
+            Thread t1 = new Thread(() -> {
+                try {
+                    barrier.await();
+                    indexOnReplica(replicationRequest, shards, replica, primaryPrimaryTerm);
+                    successFullyIndexed.set(true);
+                } catch (IllegalStateException ise) {
+                    assertThat(ise.getMessage(), either(containsString("is too old")).or(containsString("cannot be a replication target")));
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            });
+            Thread t2 = new Thread(() -> {
+                try {
+                    barrier.await();
+                    shards.promoteReplicaToPrimary(replica).get();
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            });
+            t2.start();
+            t1.start();
+            t1.join();
+            t2.join();
+
+            assertEquals(primaryPrimaryTerm + 1, replica.getPendingPrimaryTerm());
+            if (successFullyIndexed.get()) {
+                try(Translog.Snapshot snapshot = getTranslog(replica).newSnapshot()) {
+                    assertThat(snapshot.totalOperations(), equalTo(1));
+                    Translog.Operation op = snapshot.next();
+                    assertThat(op.primaryTerm(), equalTo(primaryPrimaryTerm));
+                }
+            }
+        }
+    }
+
     /**
      * test document failures (failures after seq_no generation) are added as noop operation to the translog
      * for primary and replica shards
@@ -255,7 +354,7 @@ protected EngineFactory getEngineFactory(ShardRouting routing) {
                             .source("{}", XContentType.JSON)
             );
             assertTrue(response.isFailed());
-            assertNoOpTranslogOperationForDocumentFailure(shards, 1, shards.getPrimary().getPrimaryTerm(), failureMessage);
+            assertNoOpTranslogOperationForDocumentFailure(shards, 1, shards.getPrimary().getPendingPrimaryTerm(), failureMessage);
             shards.assertAllEqual(0);
 
             // add some replicas
@@ -269,7 +368,7 @@ protected EngineFactory getEngineFactory(ShardRouting routing) {
                             .source("{}", XContentType.JSON)
             );
             assertTrue(response.isFailed());
-            assertNoOpTranslogOperationForDocumentFailure(shards, 2, shards.getPrimary().getPrimaryTerm(), failureMessage);
+            assertNoOpTranslogOperationForDocumentFailure(shards, 2, shards.getPrimary().getPendingPrimaryTerm(), failureMessage);
             shards.assertAllEqual(0);
         }
     }
@@ -361,7 +460,7 @@ public void testSeqNoCollision() throws Exception {
             // Make sure that peer-recovery transfers all but non-overridden operations.
             IndexShard replica3 = shards.addReplica();
             logger.info("--> Promote replica2 as the primary");
-            shards.promoteReplicaToPrimary(replica2);
+            shards.promoteReplicaToPrimary(replica2).get();
             logger.info("--> Recover replica3 from replica2");
             recoverReplica(replica3, replica2, true);
             try (Translog.Snapshot snapshot = getTranslog(replica3).newSnapshot()) {
diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
index f01d5e54a2e16..2d198c32ba74f 100644
--- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
@@ -245,7 +245,7 @@ public void testRecoveryAfterPrimaryPromotion() throws Exception {
                 }
             }
 
-            shards.promoteReplicaToPrimary(newPrimary);
+            shards.promoteReplicaToPrimary(newPrimary).get();
 
             // check that local checkpoint of new primary is properly tracked after primary promotion
             assertThat(newPrimary.getLocalCheckpoint(), equalTo(totalDocs - 1L));
@@ -432,7 +432,8 @@ public void testResyncAfterPrimaryPromotion() throws Exception {
                 while ((next = snapshot.next()) != null) {
                     translogOperations++;
                     assertThat("unexpected op: " + next, (int)next.seqNo(), lessThan(initialDocs + extraDocs));
-                    assertThat("unexpected primaryTerm: " + next.primaryTerm(), next.primaryTerm(), is(oldPrimary.getPrimaryTerm()));
+                    assertThat("unexpected primaryTerm: " + next.primaryTerm(), next.primaryTerm(),
+                        is(oldPrimary.getPendingPrimaryTerm()));
                     final Translog.Source source = next.getSource();
                     assertThat(source.source.utf8ToString(), is("{ \"f\": \"normal\"}"));
                 }
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
index 596575abc3025..0888dfd3c40c5 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
@@ -17,12 +17,12 @@
 
 package org.elasticsearch.index.seqno;
 
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
@@ -33,7 +33,6 @@
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.Collections;
@@ -47,7 +46,7 @@
 public class GlobalCheckpointSyncActionTests extends ESTestCase {
 
     private ThreadPool threadPool;
-    private Transport transport;
+    private CapturingTransport transport;
     private ClusterService clusterService;
     private TransportService transportService;
     private ShardStateAction shardStateAction;
@@ -57,7 +56,7 @@ public void setUp() throws Exception {
         threadPool = new TestThreadPool(getClass().getName());
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
+        transportService = transport.createCapturingTransportService(clusterService.getSettings(), threadPool,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR,  boundAddress -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java
index b2c828cb73f0c..4f4f39c614687 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java
@@ -34,11 +34,8 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -85,23 +82,14 @@ public void testBackgroundGlobalCheckpointSync() throws Exception {
                                     (MockTransportService) internalCluster().getInstance(TransportService.class, node.getName());
                             final MockTransportService receiverTransportService =
                                     (MockTransportService) internalCluster().getInstance(TransportService.class, other.getName());
-
-                            senderTransportService.addDelegate(receiverTransportService,
-                                    new MockTransportService.DelegateTransport(senderTransportService.original()) {
-                                        @Override
-                                        protected void sendRequest(
-                                                final Connection connection,
-                                                final long requestId,
-                                                final String action,
-                                                final TransportRequest request,
-                                                final TransportRequestOptions options) throws IOException {
-                                            if ("indices:admin/seq_no/global_checkpoint_sync[r]".equals(action)) {
-                                                throw new IllegalStateException("blocking indices:admin/seq_no/global_checkpoint_sync[r]");
-                                            } else {
-                                                super.sendRequest(connection, requestId, action, request, options);
-                                            }
-                                        }
-                                    });
+                            senderTransportService.addSendBehavior(receiverTransportService,
+                                (connection, requestId, action, request, options) -> {
+                                    if ("indices:admin/seq_no/global_checkpoint_sync[r]".equals(action)) {
+                                        throw new IllegalStateException("blocking indices:admin/seq_no/global_checkpoint_sync[r]");
+                                    } else {
+                                        connection.sendRequest(requestId, action, request, options);
+                                    }
+                                });
                         }
                     }
                 },
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java
index 6fdce76912e04..3948da9c1119c 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java
@@ -47,7 +47,9 @@
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
+import java.util.function.LongConsumer;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -60,7 +62,7 @@
 import static org.hamcrest.Matchers.not;
 
 public class ReplicationTrackerTests extends ESTestCase {
-
+    
     public void testEmptyShards() {
         final ReplicationTracker tracker = newTracker(AllocationId.newInitializing());
         assertThat(tracker.getGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
@@ -99,6 +101,11 @@ private static Set<String> ids(Set<AllocationId> allocationIds) {
         return allocationIds.stream().map(AllocationId::getId).collect(Collectors.toSet());
     }
 
+    private void updateLocalCheckpoint(final ReplicationTracker tracker, final String allocationId, final long localCheckpoint) {
+        tracker.updateLocalCheckpoint(allocationId, localCheckpoint);
+        assertThat(updatedGlobalCheckpoint.get(), equalTo(tracker.getGlobalCheckpoint()));
+    }
+    
     public void testGlobalCheckpointUpdate() {
         final long initialClusterStateVersion = randomNonNegativeLong();
         Map<AllocationId, Long> allocations = new HashMap<>();
@@ -137,14 +144,14 @@ public void testGlobalCheckpointUpdate() {
         assertThat(tracker.getReplicationGroup().getReplicationTargets().size(), equalTo(1));
         initializing.forEach(aId -> markAsTrackingAndInSyncQuietly(tracker, aId.getId(), NO_OPS_PERFORMED));
         assertThat(tracker.getReplicationGroup().getReplicationTargets().size(), equalTo(1 + initializing.size()));
-        allocations.keySet().forEach(aId -> tracker.updateLocalCheckpoint(aId.getId(), allocations.get(aId)));
+        allocations.keySet().forEach(aId -> updateLocalCheckpoint(tracker, aId.getId(), allocations.get(aId)));
 
         assertThat(tracker.getGlobalCheckpoint(), equalTo(minLocalCheckpoint));
 
         // increment checkpoints
         active.forEach(aId -> allocations.put(aId, allocations.get(aId) + 1 + randomInt(4)));
         initializing.forEach(aId -> allocations.put(aId, allocations.get(aId) + 1 + randomInt(4)));
-        allocations.keySet().forEach(aId -> tracker.updateLocalCheckpoint(aId.getId(), allocations.get(aId)));
+        allocations.keySet().forEach(aId -> updateLocalCheckpoint(tracker, aId.getId(), allocations.get(aId)));
 
         final long minLocalCheckpointAfterUpdates =
                 allocations.entrySet().stream().map(Map.Entry::getValue).min(Long::compareTo).orElse(UNASSIGNED_SEQ_NO);
@@ -153,7 +160,7 @@ public void testGlobalCheckpointUpdate() {
         final AllocationId extraId = AllocationId.newInitializing();
 
         // first check that adding it without the master blessing doesn't change anything.
-        tracker.updateLocalCheckpoint(extraId.getId(), minLocalCheckpointAfterUpdates + 1 + randomInt(4));
+        updateLocalCheckpoint(tracker, extraId.getId(), minLocalCheckpointAfterUpdates + 1 + randomInt(4));
         assertNull(tracker.checkpoints.get(extraId));
         expectThrows(IllegalStateException.class, () -> tracker.initiateTracking(extraId.getId()));
 
@@ -165,7 +172,7 @@ public void testGlobalCheckpointUpdate() {
 
         // now notify for the new id
         if (randomBoolean()) {
-            tracker.updateLocalCheckpoint(extraId.getId(), minLocalCheckpointAfterUpdates + 1 + randomInt(4));
+            updateLocalCheckpoint(tracker, extraId.getId(), minLocalCheckpointAfterUpdates + 1 + randomInt(4));
             markAsTrackingAndInSyncQuietly(tracker, extraId.getId(), randomInt((int) minLocalCheckpointAfterUpdates));
         } else {
             markAsTrackingAndInSyncQuietly(tracker, extraId.getId(), minLocalCheckpointAfterUpdates + 1 + randomInt(4));
@@ -175,6 +182,64 @@ public void testGlobalCheckpointUpdate() {
         assertThat(tracker.getGlobalCheckpoint(), greaterThan(minLocalCheckpoint));
     }
 
+    public void testUpdateGlobalCheckpointOnReplica() {
+        final AllocationId active = AllocationId.newInitializing();
+        final ReplicationTracker tracker = newTracker(active);
+        final long globalCheckpoint = randomLongBetween(NO_OPS_PERFORMED, Long.MAX_VALUE - 1);
+        tracker.updateGlobalCheckpointOnReplica(globalCheckpoint, "test");
+        assertThat(updatedGlobalCheckpoint.get(), equalTo(globalCheckpoint));
+        final long nonUpdate = randomLongBetween(NO_OPS_PERFORMED, globalCheckpoint);
+        updatedGlobalCheckpoint.set(UNASSIGNED_SEQ_NO);
+        tracker.updateGlobalCheckpointOnReplica(nonUpdate, "test");
+        assertThat(updatedGlobalCheckpoint.get(), equalTo(UNASSIGNED_SEQ_NO));
+        final long update = randomLongBetween(globalCheckpoint, Long.MAX_VALUE);
+        tracker.updateGlobalCheckpointOnReplica(update, "test");
+        assertThat(updatedGlobalCheckpoint.get(), equalTo(update));
+    }
+
+    public void testMarkAllocationIdAsInSync() throws BrokenBarrierException, InterruptedException {
+        final long initialClusterStateVersion = randomNonNegativeLong();
+        Map<AllocationId, Long> activeWithCheckpoints = randomAllocationsWithLocalCheckpoints(1, 1);
+        Set<AllocationId> active = new HashSet<>(activeWithCheckpoints.keySet());
+        Map<AllocationId, Long> initializingWithCheckpoints = randomAllocationsWithLocalCheckpoints(1, 1);
+        Set<AllocationId> initializing = new HashSet<>(initializingWithCheckpoints.keySet());
+        final AllocationId primaryId = active.iterator().next();
+        final AllocationId replicaId = initializing.iterator().next();
+        final ReplicationTracker tracker = newTracker(primaryId);
+        tracker.updateFromMaster(initialClusterStateVersion, ids(active), routingTable(initializing, primaryId), emptySet());
+        final long localCheckpoint = randomLongBetween(0, Long.MAX_VALUE - 1);
+        tracker.activatePrimaryMode(localCheckpoint);
+        tracker.initiateTracking(replicaId.getId());
+        final CyclicBarrier barrier = new CyclicBarrier(2);
+        final Thread thread = new Thread(() -> {
+            try {
+                barrier.await();
+                tracker.markAllocationIdAsInSync(
+                        replicaId.getId(),
+                        randomLongBetween(NO_OPS_PERFORMED, localCheckpoint - 1));
+                barrier.await();
+            } catch (BrokenBarrierException | InterruptedException e) {
+                throw new AssertionError(e);
+            }
+        });
+        thread.start();
+        barrier.await();
+        awaitBusy(tracker::pendingInSync);
+        final long updatedLocalCheckpoint = randomLongBetween(1 + localCheckpoint, Long.MAX_VALUE);
+        // there is a shard copy pending in sync, the global checkpoint can not advance
+        updatedGlobalCheckpoint.set(UNASSIGNED_SEQ_NO);
+        tracker.updateLocalCheckpoint(primaryId.getId(), updatedLocalCheckpoint);
+        assertThat(updatedGlobalCheckpoint.get(), equalTo(UNASSIGNED_SEQ_NO));
+        // we are implicitly marking the pending in sync copy as in sync with the current global checkpoint, no advancement should occur
+        tracker.updateLocalCheckpoint(replicaId.getId(), localCheckpoint);
+        assertThat(updatedGlobalCheckpoint.get(), equalTo(UNASSIGNED_SEQ_NO));
+        barrier.await();
+        thread.join();
+        // now we expect that the global checkpoint would advance
+        tracker.markAllocationIdAsInSync(replicaId.getId(), updatedLocalCheckpoint);
+        assertThat(updatedGlobalCheckpoint.get(), equalTo(updatedLocalCheckpoint));
+    }
+
     public void testMissingActiveIdsPreventAdvance() {
         final Map<AllocationId, Long> active = randomAllocationsWithLocalCheckpoints(2, 5);
         final Map<AllocationId, Long> initializing = randomAllocationsWithLocalCheckpoints(0, 5);
@@ -191,14 +256,16 @@ public void testMissingActiveIdsPreventAdvance() {
                 .entrySet()
                 .stream()
                 .filter(e -> !e.getKey().equals(missingActiveID))
-                .forEach(e -> tracker.updateLocalCheckpoint(e.getKey().getId(), e.getValue()));
+                .forEach(e -> updateLocalCheckpoint(tracker, e.getKey().getId(), e.getValue()));
 
         if (missingActiveID.equals(primaryId) == false) {
             assertThat(tracker.getGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
+            assertThat(updatedGlobalCheckpoint.get(), equalTo(UNASSIGNED_SEQ_NO));
         }
         // now update all knowledge of all shards
-        assigned.forEach((aid, localCP) -> tracker.updateLocalCheckpoint(aid.getId(), localCP));
+        assigned.forEach((aid, localCP) -> updateLocalCheckpoint(tracker, aid.getId(), localCP));
         assertThat(tracker.getGlobalCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO)));
+        assertThat(updatedGlobalCheckpoint.get(), not(equalTo(UNASSIGNED_SEQ_NO)));
     }
 
     public void testMissingInSyncIdsPreventAdvance() {
@@ -213,13 +280,15 @@ public void testMissingInSyncIdsPreventAdvance() {
         randomSubsetOf(randomIntBetween(1, initializing.size() - 1),
             initializing.keySet()).forEach(aId -> markAsTrackingAndInSyncQuietly(tracker, aId.getId(), NO_OPS_PERFORMED));
 
-        active.forEach((aid, localCP) -> tracker.updateLocalCheckpoint(aid.getId(), localCP));
+        active.forEach((aid, localCP) -> updateLocalCheckpoint(tracker, aid.getId(), localCP));
 
         assertThat(tracker.getGlobalCheckpoint(), equalTo(NO_OPS_PERFORMED));
+        assertThat(updatedGlobalCheckpoint.get(), equalTo(NO_OPS_PERFORMED));
 
         // update again
-        initializing.forEach((aid, localCP) -> tracker.updateLocalCheckpoint(aid.getId(), localCP));
+        initializing.forEach((aid, localCP) -> updateLocalCheckpoint(tracker, aid.getId(), localCP));
         assertThat(tracker.getGlobalCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO)));
+        assertThat(updatedGlobalCheckpoint.get(), not(equalTo(UNASSIGNED_SEQ_NO)));
     }
 
     public void testInSyncIdsAreIgnoredIfNotValidatedByMaster() {
@@ -236,7 +305,7 @@ public void testInSyncIdsAreIgnoredIfNotValidatedByMaster() {
 
         List<Map<AllocationId, Long>> allocations = Arrays.asList(active, initializing, nonApproved);
         Collections.shuffle(allocations, random());
-        allocations.forEach(a -> a.forEach((aid, localCP) -> tracker.updateLocalCheckpoint(aid.getId(), localCP)));
+        allocations.forEach(a -> a.forEach((aid, localCP) -> updateLocalCheckpoint(tracker, aid.getId(), localCP)));
 
         assertThat(tracker.getGlobalCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO)));
     }
@@ -271,7 +340,7 @@ public void testInSyncIdsAreRemovedIfNotValidatedByMaster() {
             initializing.forEach(k -> markAsTrackingAndInSyncQuietly(tracker, k.getId(), NO_OPS_PERFORMED));
         }
         if (randomBoolean()) {
-            allocations.forEach((aid, localCP) -> tracker.updateLocalCheckpoint(aid.getId(), localCP));
+            allocations.forEach((aid, localCP) -> updateLocalCheckpoint(tracker, aid.getId(), localCP));
         }
 
         // now remove shards
@@ -281,9 +350,9 @@ public void testInSyncIdsAreRemovedIfNotValidatedByMaster() {
                     ids(activeToStay.keySet()),
                     routingTable(initializingToStay.keySet(), primaryId),
                     emptySet());
-            allocations.forEach((aid, ckp) -> tracker.updateLocalCheckpoint(aid.getId(), ckp + 10L));
+            allocations.forEach((aid, ckp) -> updateLocalCheckpoint(tracker, aid.getId(), ckp + 10L));
         } else {
-            allocations.forEach((aid, ckp) -> tracker.updateLocalCheckpoint(aid.getId(), ckp + 10L));
+            allocations.forEach((aid, ckp) -> updateLocalCheckpoint(tracker, aid.getId(), ckp + 10L));
             tracker.updateFromMaster(
                     initialClusterStateVersion + 2,
                     ids(activeToStay.keySet()),
@@ -331,7 +400,7 @@ public void testWaitForAllocationIdToBeInSync() throws Exception {
         final List<Integer> elements = IntStream.rangeClosed(0, globalCheckpoint - 1).boxed().collect(Collectors.toList());
         Randomness.shuffle(elements);
         for (int i = 0; i < elements.size(); i++) {
-            tracker.updateLocalCheckpoint(trackingAllocationId.getId(), elements.get(i));
+            updateLocalCheckpoint(tracker, trackingAllocationId.getId(), elements.get(i));
             assertFalse(complete.get());
             assertFalse(tracker.getTrackedLocalCheckpointForShard(trackingAllocationId.getId()).inSync);
             assertBusy(() -> assertTrue(tracker.pendingInSync.contains(trackingAllocationId.getId())));
@@ -339,7 +408,7 @@ public void testWaitForAllocationIdToBeInSync() throws Exception {
 
         if (randomBoolean()) {
             // normal path, shard catches up
-            tracker.updateLocalCheckpoint(trackingAllocationId.getId(), randomIntBetween(globalCheckpoint, 64));
+            updateLocalCheckpoint(tracker, trackingAllocationId.getId(), randomIntBetween(globalCheckpoint, 64));
             // synchronize with the waiting thread to mark that it is complete
             barrier.await();
             assertTrue(complete.get());
@@ -355,13 +424,16 @@ public void testWaitForAllocationIdToBeInSync() throws Exception {
         assertFalse(tracker.pendingInSync.contains(trackingAllocationId.getId()));
         thread.join();
     }
+    
+    private AtomicLong updatedGlobalCheckpoint = new AtomicLong(UNASSIGNED_SEQ_NO);
 
     private ReplicationTracker newTracker(final AllocationId allocationId) {
         return new ReplicationTracker(
                 new ShardId("test", "_na_", 0),
                 allocationId.getId(),
                 IndexSettingsModule.newIndexSettings("test", Settings.EMPTY),
-                UNASSIGNED_SEQ_NO);
+                UNASSIGNED_SEQ_NO,
+                updatedGlobalCheckpoint::set);
     }
 
     public void testWaitForAllocationIdToBeInSyncCanBeInterrupted() throws BrokenBarrierException, InterruptedException {
@@ -488,10 +560,10 @@ public void testUpdateAllocationIdsFromMaster() throws Exception {
         // the tracking allocation IDs should play no role in determining the global checkpoint
         final Map<AllocationId, Integer> activeLocalCheckpoints =
                 newActiveAllocationIds.stream().collect(Collectors.toMap(Function.identity(), a -> randomIntBetween(1, 1024)));
-        activeLocalCheckpoints.forEach((a, l) -> tracker.updateLocalCheckpoint(a.getId(), l));
+        activeLocalCheckpoints.forEach((a, l) -> updateLocalCheckpoint(tracker, a.getId(), l));
         final Map<AllocationId, Integer> initializingLocalCheckpoints =
                 newInitializingAllocationIds.stream().collect(Collectors.toMap(Function.identity(), a -> randomIntBetween(1, 1024)));
-        initializingLocalCheckpoints.forEach((a, l) -> tracker.updateLocalCheckpoint(a.getId(), l));
+        initializingLocalCheckpoints.forEach((a, l) -> updateLocalCheckpoint(tracker, a.getId(), l));
         assertTrue(
                 activeLocalCheckpoints
                         .entrySet()
@@ -504,6 +576,7 @@ public void testUpdateAllocationIdsFromMaster() throws Exception {
                         .allMatch(e -> tracker.getTrackedLocalCheckpointForShard(e.getKey().getId()).getLocalCheckpoint() == e.getValue()));
         final long minimumActiveLocalCheckpoint = (long) activeLocalCheckpoints.values().stream().min(Integer::compareTo).get();
         assertThat(tracker.getGlobalCheckpoint(), equalTo(minimumActiveLocalCheckpoint));
+        assertThat(updatedGlobalCheckpoint.get(), equalTo(minimumActiveLocalCheckpoint));
         final long minimumInitailizingLocalCheckpoint = (long) initializingLocalCheckpoints.values().stream().min(Integer::compareTo).get();
 
         // now we are going to add a new allocation ID and bring it in sync which should move it to the in-sync allocation IDs
@@ -635,10 +708,11 @@ public void testPrimaryContextHandoff() throws IOException {
 
         FakeClusterState clusterState = initialState();
         final AllocationId primaryAllocationId = clusterState.routingTable.primaryShard().allocationId();
+        final LongConsumer onUpdate = updatedGlobalCheckpoint -> {};
         ReplicationTracker oldPrimary =
-                new ReplicationTracker(shardId, primaryAllocationId.getId(), indexSettings, UNASSIGNED_SEQ_NO);
+                new ReplicationTracker(shardId, primaryAllocationId.getId(), indexSettings, UNASSIGNED_SEQ_NO, onUpdate);
         ReplicationTracker newPrimary =
-                new ReplicationTracker(shardId, primaryAllocationId.getRelocationId(), indexSettings, UNASSIGNED_SEQ_NO);
+                new ReplicationTracker(shardId, primaryAllocationId.getRelocationId(), indexSettings, UNASSIGNED_SEQ_NO, onUpdate);
 
         Set<String> allocationIds = new HashSet<>(Arrays.asList(oldPrimary.shardAllocationId, newPrimary.shardAllocationId));
 
@@ -770,8 +844,10 @@ public void testPrimaryContextHandoff() throws IOException {
         assertThat(newPrimary.routingTable, equalTo(oldPrimary.routingTable));
         assertThat(newPrimary.replicationGroup, equalTo(oldPrimary.replicationGroup));
 
+        assertFalse(oldPrimary.relocated);
         oldPrimary.completeRelocationHandoff();
         assertFalse(oldPrimary.primaryMode);
+        assertTrue(oldPrimary.relocated);
     }
 
     public void testIllegalStateExceptionIfUnknownAllocationId() {
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java
index 27d08b76c0310..23337def2ae1b 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java
@@ -71,7 +71,7 @@ public class IndexShardOperationPermitsTests extends ESTestCase {
     public static void setupThreadPool() {
         int writeThreadPoolSize = randomIntBetween(1, 2);
         int writeThreadPoolQueueSize = randomIntBetween(1, 2);
-        threadPool = new TestThreadPool("IndexShardOperationsLockTests",
+        threadPool = new TestThreadPool("IndexShardOperationPermitsTests",
             Settings.builder()
                 .put("thread_pool." + ThreadPool.Names.WRITE + ".size", writeThreadPoolSize)
                 .put("thread_pool." + ThreadPool.Names.WRITE + ".queue_size", writeThreadPoolQueueSize)
@@ -100,7 +100,7 @@ public void checkNoInflightOperations() {
         assertThat(permits.getActiveOperationsCount(), equalTo(0));
     }
 
-    public void testAllOperationsInvoked() throws InterruptedException, TimeoutException, ExecutionException {
+    public void testAllOperationsInvoked() throws InterruptedException, TimeoutException {
         int numThreads = 10;
 
         class DummyException extends RuntimeException {}
@@ -187,7 +187,7 @@ public void testOperationsInvokedImmediatelyIfNoBlock() throws ExecutionExceptio
         future.get().close();
     }
 
-    public void testOperationsIfClosed() throws ExecutionException, InterruptedException {
+    public void testOperationsIfClosed() {
         PlainActionFuture<Releasable> future = new PlainActionFuture<>();
         permits.close();
         permits.acquire(future, ThreadPool.Names.GENERIC, true, "");
@@ -195,10 +195,12 @@ public void testOperationsIfClosed() throws ExecutionException, InterruptedExcep
         assertThat(exception.getCause(), instanceOf(IndexShardClosedException.class));
     }
 
-    public void testBlockIfClosed() throws ExecutionException, InterruptedException {
+    public void testBlockIfClosed() {
         permits.close();
         expectThrows(IndexShardClosedException.class, () -> permits.blockOperations(randomInt(10), TimeUnit.MINUTES,
             () -> { throw new IllegalArgumentException("fake error"); }));
+        expectThrows(IndexShardClosedException.class, () -> permits.asyncBlockOperations(randomInt(10), TimeUnit.MINUTES,
+            () -> { throw new IllegalArgumentException("fake error"); }, e -> { throw new AssertionError(e); }));
     }
 
     public void testOperationsDelayedIfBlock() throws ExecutionException, InterruptedException, TimeoutException {
@@ -210,6 +212,36 @@ public void testOperationsDelayedIfBlock() throws ExecutionException, Interrupte
         future.get(1, TimeUnit.HOURS).close();
     }
 
+    public void testGetBlockWhenBlocked() throws ExecutionException, InterruptedException, TimeoutException {
+        PlainActionFuture<Releasable> future = new PlainActionFuture<>();
+        final CountDownLatch blockAcquired = new CountDownLatch(1);
+        final CountDownLatch releaseBlock = new CountDownLatch(1);
+        final AtomicBoolean blocked = new AtomicBoolean();
+        try (Releasable ignored = blockAndWait()) {
+            permits.acquire(future, ThreadPool.Names.GENERIC, true, "");
+
+            permits.asyncBlockOperations(
+                30,
+                TimeUnit.MINUTES,
+                () -> {
+                    blocked.set(true);
+                    blockAcquired.countDown();
+                    releaseBlock.await();
+                },
+                e -> {
+                    throw new RuntimeException(e);
+                });
+            assertFalse(blocked.get());
+            assertFalse(future.isDone());
+        }
+        blockAcquired.await();
+        assertTrue(blocked.get());
+        assertFalse(future.isDone());
+        releaseBlock.countDown();
+
+        future.get(1, TimeUnit.HOURS).close();
+    }
+
     /**
      * Tests that the ThreadContext is restored when a operation is executed after it has been delayed due to a block
      */
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 1880b6b0954dc..67a28c9b33623 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -31,6 +31,7 @@
 import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.Assertions;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
@@ -297,7 +298,7 @@ public void testClosesPreventsNewOperations() throws InterruptedException, Execu
             // expected
         }
         try {
-            indexShard.acquireReplicaOperationPermit(indexShard.getPrimaryTerm(), SequenceNumbers.UNASSIGNED_SEQ_NO, null,
+            indexShard.acquireReplicaOperationPermit(indexShard.getPendingPrimaryTerm(), SequenceNumbers.UNASSIGNED_SEQ_NO, null,
                 ThreadPool.Names.WRITE, "");
             fail("we should not be able to increment anymore");
         } catch (IndexShardClosedException e) {
@@ -308,7 +309,7 @@ public void testClosesPreventsNewOperations() throws InterruptedException, Execu
     public void testRejectOperationPermitWithHigherTermWhenNotStarted() throws IOException {
         IndexShard indexShard = newShard(false);
         expectThrows(IndexShardNotStartedException.class, () ->
-            indexShard.acquireReplicaOperationPermit(indexShard.getPrimaryTerm() + randomIntBetween(1, 100),
+            indexShard.acquireReplicaOperationPermit(indexShard.getPendingPrimaryTerm() + randomIntBetween(1, 100),
                 SequenceNumbers.UNASSIGNED_SEQ_NO, null, ThreadPool.Names.WRITE, ""));
         closeShards(indexShard);
     }
@@ -331,7 +332,7 @@ public void testPrimaryPromotionDelaysOperations() throws IOException, BrokenBar
                     throw new RuntimeException(e);
                 }
                 indexShard.acquireReplicaOperationPermit(
-                        indexShard.getPrimaryTerm(),
+                        indexShard.getPendingPrimaryTerm(),
                         indexShard.getGlobalCheckpoint(),
                         new ActionListener<Releasable>() {
                             @Override
@@ -418,16 +419,13 @@ public void onFailure(Exception e) {
     }
 
     /**
-     * This test makes sure that people can use the shard routing entry to check whether a shard was already promoted to
-     * a primary. Concretely this means, that when we publish the routing entry via {@link IndexShard#routingEntry()} the following
-     * should have happened
-     * 1) Internal state (ala ReplicationTracker) have been updated
-     * 2) Primary term is set to the new term
+     * This test makes sure that people can use the shard routing entry + take an operation permit to check whether a shard was already
+     * promoted to a primary.
      */
     public void testPublishingOrderOnPromotion() throws IOException, InterruptedException, BrokenBarrierException {
         final IndexShard indexShard = newShard(false);
         recoveryEmptyReplica(indexShard, randomBoolean());
-        final long promotedTerm = indexShard.getPrimaryTerm() + 1;
+        final long promotedTerm = indexShard.getPendingPrimaryTerm() + 1;
         final CyclicBarrier barrier = new CyclicBarrier(2);
         final AtomicBoolean stop = new AtomicBoolean();
         final Thread thread = new Thread(() -> {
@@ -438,8 +436,12 @@ public void testPublishingOrderOnPromotion() throws IOException, InterruptedExce
             }
             while(stop.get() == false) {
                 if (indexShard.routingEntry().primary()) {
-                    assertThat(indexShard.getPrimaryTerm(), equalTo(promotedTerm));
-                    assertThat(indexShard.getReplicationGroup(), notNullValue());
+                    assertThat(indexShard.getPendingPrimaryTerm(), equalTo(promotedTerm));
+                    final PlainActionFuture<Releasable> permitAcquiredFuture = new PlainActionFuture<>();
+                    indexShard.acquirePrimaryOperationPermit(permitAcquiredFuture, ThreadPool.Names.SAME, "bla");
+                    try (Releasable ignored = permitAcquiredFuture.actionGet()) {
+                        assertThat(indexShard.getReplicationGroup(), notNullValue());
+                    }
                 }
             }
         });
@@ -504,7 +506,7 @@ public void testPrimaryPromotionRollsGeneration() throws Exception {
 
         // promote the replica
         final ShardRouting replicaRouting = indexShard.routingEntry();
-        final long newPrimaryTerm = indexShard.getPrimaryTerm() + between(1, 10000);
+        final long newPrimaryTerm = indexShard.getPendingPrimaryTerm() + between(1, 10000);
         final ShardRouting primaryRouting =
                 newShardRouting(
                         replicaRouting.shardId(),
@@ -558,26 +560,21 @@ public void testOperationPermitsOnPrimaryShards() throws InterruptedException, E
             ShardRouting replicaRouting = indexShard.routingEntry();
             ShardRouting primaryRouting = newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), null,
                 true, ShardRoutingState.STARTED, replicaRouting.allocationId());
-            final long newPrimaryTerm = indexShard.getPrimaryTerm() + between(1, 1000);
+            final long newPrimaryTerm = indexShard.getPendingPrimaryTerm() + between(1, 1000);
+            CountDownLatch latch = new CountDownLatch(1);
             indexShard.updateShardState(primaryRouting, newPrimaryTerm, (shard, listener) -> {
                     assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(newPrimaryTerm));
+                    latch.countDown();
                 }, 0L,
                 Collections.singleton(indexShard.routingEntry().allocationId().getId()),
                 new IndexShardRoutingTable.Builder(indexShard.shardId()).addShard(primaryRouting).build(),
                 Collections.emptySet());
+            latch.await();
         } else {
             indexShard = newStartedShard(true);
         }
-        final long primaryTerm = indexShard.getPrimaryTerm();
+        final long primaryTerm = indexShard.getPendingPrimaryTerm();
         assertEquals(0, indexShard.getActiveOperationsCount());
-        if (indexShard.routingEntry().isRelocationTarget() == false) {
-            try {
-                indexShard.acquireReplicaOperationPermit(primaryTerm, indexShard.getGlobalCheckpoint(), null, ThreadPool.Names.WRITE, "");
-                fail("shard shouldn't accept operations as replica");
-            } catch (IllegalStateException ignored) {
-
-            }
-        }
         Releasable operation1 = acquirePrimaryOperationPermitBlockingly(indexShard);
         assertEquals(1, indexShard.getActiveOperationsCount());
         Releasable operation2 = acquirePrimaryOperationPermitBlockingly(indexShard);
@@ -586,6 +583,22 @@ public void testOperationPermitsOnPrimaryShards() throws InterruptedException, E
         Releasables.close(operation1, operation2);
         assertEquals(0, indexShard.getActiveOperationsCount());
 
+        if (Assertions.ENABLED && indexShard.routingEntry().isRelocationTarget() == false) {
+            assertThat(expectThrows(AssertionError.class, () -> indexShard.acquireReplicaOperationPermit(primaryTerm,
+                indexShard.getGlobalCheckpoint(), new ActionListener<Releasable>() {
+                    @Override
+                    public void onResponse(Releasable releasable) {
+                        fail();
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        fail();
+                    }
+                },
+                ThreadPool.Names.WRITE, "")).getMessage(), containsString("in primary mode cannot be a replication target"));
+        }
+
         closeShards(indexShard);
     }
 
@@ -643,14 +656,14 @@ public void testOperationPermitOnReplicaShards() throws Exception {
         logger.info("shard routing to {}", shardRouting);
 
         assertEquals(0, indexShard.getActiveOperationsCount());
-        if (shardRouting.primary() == false) {
-            final IllegalStateException e =
-                    expectThrows(IllegalStateException.class,
+        if (shardRouting.primary() == false && Assertions.ENABLED) {
+            final AssertionError e =
+                    expectThrows(AssertionError.class,
                         () -> indexShard.acquirePrimaryOperationPermit(null, ThreadPool.Names.WRITE, ""));
-            assertThat(e, hasToString(containsString("shard " + shardRouting + " is not a primary")));
+            assertThat(e, hasToString(containsString("acquirePrimaryOperationPermit should only be called on primary shard")));
         }
 
-        final long primaryTerm = indexShard.getPrimaryTerm();
+        final long primaryTerm = indexShard.getPendingPrimaryTerm();
         final long translogGen = engineClosed ? -1 : getTranslog(indexShard).getGeneration().translogFileGeneration;
 
         final Releasable operation1;
@@ -728,7 +741,7 @@ public void onFailure(Exception e) {
                 ActionListener<Releasable> listener = new ActionListener<Releasable>() {
                     @Override
                     public void onResponse(Releasable releasable) {
-                        assertThat(indexShard.getPrimaryTerm(), equalTo(newPrimaryTerm));
+                        assertThat(indexShard.getPendingPrimaryTerm(), equalTo(newPrimaryTerm));
                         assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(newPrimaryTerm));
                         assertThat(indexShard.getLocalCheckpoint(), equalTo(expectedLocalCheckpoint));
                         assertThat(indexShard.getGlobalCheckpoint(), equalTo(newGlobalCheckPoint));
@@ -765,7 +778,7 @@ private void finish() {
             barrier.await();
             if (indexShard.state() == IndexShardState.CREATED || indexShard.state() == IndexShardState.RECOVERING) {
                 barrier.await();
-                assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm));
+                assertThat(indexShard.getPendingPrimaryTerm(), equalTo(primaryTerm));
                 assertFalse(onResponse.get());
                 assertThat(onFailure.get(), instanceOf(IndexShardNotStartedException.class));
                 Releasables.close(operation1);
@@ -774,18 +787,19 @@ private void finish() {
                 // our operation should be blocked until the previous operations complete
                 assertFalse(onResponse.get());
                 assertNull(onFailure.get());
-                assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm));
+                assertThat(indexShard.operationPrimaryTerm, equalTo(primaryTerm));
                 assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(primaryTerm));
                 Releasables.close(operation1);
                 // our operation should still be blocked
                 assertFalse(onResponse.get());
                 assertNull(onFailure.get());
-                assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm));
+                assertThat(indexShard.operationPrimaryTerm, equalTo(primaryTerm));
                 assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(primaryTerm));
                 Releasables.close(operation2);
                 barrier.await();
                 // now lock acquisition should have succeeded
-                assertThat(indexShard.getPrimaryTerm(), equalTo(newPrimaryTerm));
+                assertThat(indexShard.operationPrimaryTerm, equalTo(newPrimaryTerm));
+                assertThat(indexShard.getPendingPrimaryTerm(), equalTo(newPrimaryTerm));
                 assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(newPrimaryTerm));
                 if (engineClosed) {
                     assertFalse(onResponse.get());
@@ -884,7 +898,7 @@ public void testRestoreLocalCheckpointTrackerFromTranslogOnPromotion() throws IO
 
         final CountDownLatch latch = new CountDownLatch(1);
         indexShard.acquireReplicaOperationPermit(
-                indexShard.getPrimaryTerm() + 1,
+                indexShard.getPendingPrimaryTerm() + 1,
                 globalCheckpoint,
                 new ActionListener<Releasable>() {
                     @Override
@@ -906,7 +920,7 @@ public void onFailure(Exception e) {
         final CountDownLatch resyncLatch = new CountDownLatch(1);
         indexShard.updateShardState(
                 newRouting,
-                indexShard.getPrimaryTerm() + 1,
+                indexShard.getPendingPrimaryTerm() + 1,
                 (s, r) -> resyncLatch.countDown(),
                 1L,
                 Collections.singleton(newRouting.allocationId().getId()),
@@ -938,7 +952,7 @@ public void testThrowBackLocalCheckpointOnReplica() throws IOException, Interrup
                         Math.toIntExact(indexShard.getLocalCheckpoint()));
         final CountDownLatch latch = new CountDownLatch(1);
         indexShard.acquireReplicaOperationPermit(
-                indexShard.primaryTerm + 1,
+                indexShard.pendingPrimaryTerm + 1,
                 globalCheckpoint,
                 new ActionListener<Releasable>() {
                     @Override
@@ -975,7 +989,7 @@ public void testConcurrentTermIncreaseOnReplicaShard() throws BrokenBarrierExcep
         final CyclicBarrier barrier = new CyclicBarrier(3);
         final CountDownLatch latch = new CountDownLatch(2);
 
-        final long primaryTerm = indexShard.getPrimaryTerm();
+        final long primaryTerm = indexShard.getPendingPrimaryTerm();
         final AtomicLong counter = new AtomicLong();
         final AtomicReference<Exception> onFailure = new AtomicReference<>();
 
@@ -993,7 +1007,7 @@ public void testConcurrentTermIncreaseOnReplicaShard() throws BrokenBarrierExcep
                         @Override
                         public void onResponse(Releasable releasable) {
                             counter.incrementAndGet();
-                            assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm + increment));
+                            assertThat(indexShard.getPendingPrimaryTerm(), equalTo(primaryTerm + increment));
                             latch.countDown();
                             releasable.close();
                         }
@@ -1037,7 +1051,7 @@ public void onFailure(Exception e) {
             assertThat(counter.get(), equalTo(2L));
         }
 
-        assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm + Math.max(firstIncrement, secondIncrement)));
+        assertThat(indexShard.getPendingPrimaryTerm(), equalTo(primaryTerm + Math.max(firstIncrement, secondIncrement)));
 
         closeShards(indexShard);
     }
@@ -1416,14 +1430,14 @@ public void testLockingBeforeAndAfterRelocated() throws Exception {
             recoveryThread.start();
             latch.await();
             // recovery can only be finalized after we release the current primaryOperationLock
-            assertTrue(shard.isPrimaryMode());
+            assertFalse(shard.isRelocatedPrimary());
         }
         // recovery can be now finalized
         recoveryThread.join();
-        assertFalse(shard.isPrimaryMode());
+        assertTrue(shard.isRelocatedPrimary());
         try (Releasable ignored = acquirePrimaryOperationPermitBlockingly(shard)) {
             // lock can again be acquired
-            assertFalse(shard.isPrimaryMode());
+            assertTrue(shard.isRelocatedPrimary());
         }
 
         closeShards(shard);
@@ -1465,7 +1479,7 @@ public void onResponse(Releasable releasable) {
 
     public void testStressRelocated() throws Exception {
         final IndexShard shard = newStartedShard(true);
-        assertTrue(shard.isPrimaryMode());
+        assertFalse(shard.isRelocatedPrimary());
         IndexShardTestCase.updateRoutingEntry(shard, ShardRoutingHelper.relocate(shard.routingEntry(), "other_node"));
         final int numThreads = randomIntBetween(2, 4);
         Thread[] indexThreads = new Thread[numThreads];
@@ -1501,14 +1515,14 @@ public void run() {
         assertThat(relocated.get(), equalTo(false));
         assertThat(shard.getActiveOperationsCount(), greaterThan(0));
         // ensure we only transition after pending operations completed
-        assertTrue(shard.isPrimaryMode());
+        assertFalse(shard.isRelocatedPrimary());
         // complete pending operations
         barrier.await();
         // complete recovery/relocation
         recoveryThread.join();
         // ensure relocated successfully once pending operations are done
         assertThat(relocated.get(), equalTo(true));
-        assertFalse(shard.isPrimaryMode());
+        assertTrue(shard.isRelocatedPrimary());
         assertThat(shard.getActiveOperationsCount(), equalTo(0));
 
         for (Thread indexThread : indexThreads) {
@@ -1572,7 +1586,7 @@ protected void doRun() throws Exception {
         cyclicBarrier.await();
         relocationThread.join();
         cancellingThread.join();
-        if (shard.isPrimaryMode() == false) {
+        if (shard.isRelocatedPrimary()) {
             logger.debug("shard was relocated successfully");
             assertThat(cancellingException.get(), instanceOf(IllegalIndexShardStateException.class));
             assertThat("current routing:" + shard.routingEntry(), shard.routingEntry().relocating(), equalTo(true));
@@ -1719,7 +1733,7 @@ public void testRecoverFromStoreWithNoOps() throws IOException {
             while ((operation = snapshot.next()) != null) {
                 if (operation.opType() == Translog.Operation.Type.NO_OP) {
                     numNoops++;
-                    assertEquals(newShard.getPrimaryTerm(), operation.primaryTerm());
+                    assertEquals(newShard.getPendingPrimaryTerm(), operation.primaryTerm());
                     assertEquals(0, operation.seqNo());
                 }
             }
@@ -1826,7 +1840,7 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception {
         flushShard(shard);
         assertThat(getShardDocUIDs(shard), containsInAnyOrder("doc-0", "doc-1"));
         // Simulate resync (without rollback): Noop #1, index #2
-        acquireReplicaOperationPermitBlockingly(shard, shard.primaryTerm + 1);
+        acquireReplicaOperationPermitBlockingly(shard, shard.pendingPrimaryTerm + 1);
         shard.markSeqNoAsNoop(1, "test");
         shard.applyIndexOperationOnReplica(2, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
             SourceToParse.source(indexName, "_doc", "doc-2", new BytesArray("{}"), XContentType.JSON));
@@ -1837,7 +1851,8 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception {
         IndexShard newShard = reinitShard(shard,
             newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING,
                 RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE));
-        newShard.primaryTerm++;
+        newShard.pendingPrimaryTerm++;
+        newShard.operationPrimaryTerm++;
         DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
         assertTrue(newShard.recoverFromStore());
@@ -1852,7 +1867,7 @@ public void testRecoveryFailsAfterMovingToRelocatedState() throws InterruptedExc
         ShardRouting inRecoveryRouting = ShardRoutingHelper.relocate(origRouting, "some_node");
         IndexShardTestCase.updateRoutingEntry(shard, inRecoveryRouting);
         shard.relocated(primaryContext -> {});
-        assertFalse(shard.isPrimaryMode());
+        assertTrue(shard.isRelocatedPrimary());
         try {
             IndexShardTestCase.updateRoutingEntry(shard, origRouting);
             fail("Expected IndexShardRelocatedException");
@@ -2160,11 +2175,11 @@ public void testRecoverFromTranslog() throws IOException {
         int numCorruptEntries = 0;
         for (int i = 0; i < numTotalEntries; i++) {
             if (randomBoolean()) {
-                operations.add(new Translog.Index("_doc", "1", 0, primary.getPrimaryTerm(), 1,
+                operations.add(new Translog.Index("_doc", "1", 0, primary.getPendingPrimaryTerm(), 1,
                     "{\"foo\" : \"bar\"}".getBytes(Charset.forName("UTF-8")), null, -1));
             } else {
                 // corrupt entry
-                operations.add(new Translog.Index("_doc", "2", 1,  primary.getPrimaryTerm(), 1,
+                operations.add(new Translog.Index("_doc", "2", 1,  primary.getPendingPrimaryTerm(), 1,
                     "{\"foo\" : \"bar}".getBytes(Charset.forName("UTF-8")), null, -1));
                 numCorruptEntries++;
             }
@@ -2381,9 +2396,7 @@ public void testDocStats() throws IOException {
         try {
             indexShard = newStartedShard();
             final long numDocs = randomIntBetween(2, 32); // at least two documents so we have docs to delete
-            // Delete at least numDocs/10 documents otherwise the number of deleted docs will be below 10%
-            // and forceMerge will refuse to expunge deletes
-            final long numDocsToDelete = randomIntBetween((int) Math.ceil(Math.nextUp(numDocs / 10.0)), Math.toIntExact(numDocs));
+            final long numDocsToDelete = randomLongBetween(1, numDocs);
             for (int i = 0; i < numDocs; i++) {
                 final String id = Integer.toString(i);
                 indexDoc(indexShard, "_doc", id);
@@ -2433,7 +2446,6 @@ public void testDocStats() throws IOException {
 
             // merge them away
             final ForceMergeRequest forceMergeRequest = new ForceMergeRequest();
-            forceMergeRequest.onlyExpungeDeletes(randomBoolean());
             forceMergeRequest.maxNumSegments(1);
             indexShard.forceMerge(forceMergeRequest);
 
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
index 037b5d68fb865..7baae06b67151 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
@@ -148,7 +148,7 @@ public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
         ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null);
         Engine.Delete delete = new Engine.Delete("test", "1", new Term("_id", Uid.encodeId(doc.id())), randomNonNegativeLong());
         Engine.Index index = new Engine.Index(new Term("_id", Uid.encodeId(doc.id())), randomNonNegativeLong(), doc);
-        compositeListener.postDelete(randomShardId, delete, new Engine.DeleteResult(1, SequenceNumbers.UNASSIGNED_SEQ_NO, true));
+        compositeListener.postDelete(randomShardId, delete, new Engine.DeleteResult(1, 0, SequenceNumbers.UNASSIGNED_SEQ_NO, true));
         assertEquals(0, preIndex.get());
         assertEquals(0, postIndex.get());
         assertEquals(0, postIndexException.get());
@@ -172,7 +172,7 @@ public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
         assertEquals(2, postDelete.get());
         assertEquals(2, postDeleteException.get());
 
-        compositeListener.postIndex(randomShardId, index, new Engine.IndexResult(0, SequenceNumbers.UNASSIGNED_SEQ_NO, false));
+        compositeListener.postIndex(randomShardId, index, new Engine.IndexResult(0, 0, SequenceNumbers.UNASSIGNED_SEQ_NO, false));
         assertEquals(0, preIndex.get());
         assertEquals(2, postIndex.get());
         assertEquals(0, postIndexException.get());
diff --git a/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java b/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
index 4444f475329b3..ae2cc84e4870c 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
@@ -83,7 +83,7 @@ public void testSyncerSendsOffCorrectDocuments() throws Exception {
         boolean syncNeeded = numDocs > 0;
 
         String allocationId = shard.routingEntry().allocationId().getId();
-        shard.updateShardState(shard.routingEntry(), shard.getPrimaryTerm(), null, 1000L, Collections.singleton(allocationId),
+        shard.updateShardState(shard.routingEntry(), shard.getPendingPrimaryTerm(), null, 1000L, Collections.singleton(allocationId),
             new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build(), Collections.emptySet());
         shard.updateLocalCheckpointForShard(allocationId, globalCheckPoint);
         assertEquals(globalCheckPoint, shard.getGlobalCheckpoint());
@@ -142,7 +142,7 @@ public void testSyncerOnClosingShard() throws Exception {
         }
 
         String allocationId = shard.routingEntry().allocationId().getId();
-        shard.updateShardState(shard.routingEntry(), shard.getPrimaryTerm(), null, 1000L, Collections.singleton(allocationId),
+        shard.updateShardState(shard.routingEntry(), shard.getPendingPrimaryTerm(), null, 1000L, Collections.singleton(allocationId),
             new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build(), Collections.emptySet());
 
         CountDownLatch syncCalledLatch = new CountDownLatch(1);
diff --git a/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
index d3119bbc0fdc4..fdadb5999f77d 100644
--- a/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -70,8 +70,6 @@
 import org.elasticsearch.test.MockIndexEventListener;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
@@ -338,19 +336,15 @@ public void testCorruptionOnNetworkLayerFinalizingRecovery() throws ExecutionExc
         final CountDownLatch hasCorrupted = new CountDownLatch(1);
         for (NodeStats dataNode : dataNodeStats) {
             MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
-            mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {
-
-                @Override
-                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
-                    if (corrupt.get() && action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
-                        RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
-                        byte[] array = BytesRef.deepCopyOf(req.content().toBytesRef()).bytes;
-                        int i = randomIntBetween(0, req.content().length() - 1);
-                        array[i] = (byte) ~array[i]; // flip one byte in the content
-                        hasCorrupted.countDown();
-                    }
-                    super.sendRequest(connection, requestId, action, request, options);
+            mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), (connection, requestId, action, request, options) -> {
+                if (corrupt.get() && action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
+                    RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
+                    byte[] array = BytesRef.deepCopyOf(req.content().toBytesRef()).bytes;
+                    int i = randomIntBetween(0, req.content().length() - 1);
+                    array[i] = (byte) ~array[i]; // flip one byte in the content
+                    hasCorrupted.countDown();
                 }
+                connection.sendRequest(requestId, action, request, options);
             });
         }
 
@@ -410,25 +404,21 @@ public void testCorruptionOnNetworkLayer() throws ExecutionException, Interrupte
         final boolean truncate = randomBoolean();
         for (NodeStats dataNode : dataNodeStats) {
             MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
-            mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {
-
-                @Override
-                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
-                    if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
-                        RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
-                        if (truncate && req.length() > 1) {
-                            BytesRef bytesRef = req.content().toBytesRef();
-                            BytesArray array = new BytesArray(bytesRef.bytes, bytesRef.offset, (int) req.length() - 1);
-                            request = new RecoveryFileChunkRequest(req.recoveryId(), req.shardId(), req.metadata(), req.position(), array, req.lastChunk(), req.totalTranslogOps(), req.sourceThrottleTimeInNanos());
-                        } else {
-                            assert req.content().toBytesRef().bytes == req.content().toBytesRef().bytes : "no internal reference!!";
-                            final byte[] array = req.content().toBytesRef().bytes;
-                            int i = randomIntBetween(0, req.content().length() - 1);
-                            array[i] = (byte) ~array[i]; // flip one byte in the content
-                        }
+            mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), (connection, requestId, action, request, options) -> {
+                if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
+                    RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
+                    if (truncate && req.length() > 1) {
+                        BytesRef bytesRef = req.content().toBytesRef();
+                        BytesArray array = new BytesArray(bytesRef.bytes, bytesRef.offset, (int) req.length() - 1);
+                        request = new RecoveryFileChunkRequest(req.recoveryId(), req.shardId(), req.metadata(), req.position(), array, req.lastChunk(), req.totalTranslogOps(), req.sourceThrottleTimeInNanos());
+                    } else {
+                        assert req.content().toBytesRef().bytes == req.content().toBytesRef().bytes : "no internal reference!!";
+                        final byte[] array = req.content().toBytesRef().bytes;
+                        int i = randomIntBetween(0, req.content().length() - 1);
+                        array[i] = (byte) ~array[i]; // flip one byte in the content
                     }
-                    super.sendRequest(connection, requestId, action, request, options);
                 }
+                connection.sendRequest(requestId, action, request, options);
             });
         }
 
diff --git a/server/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/server/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
index 9cc6d86bc2fba..7d548fc42d695 100644
--- a/server/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
+++ b/server/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.test.transport.MockTransportService;
 
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collection;
@@ -86,7 +87,7 @@ public void testCorruptTranslogFiles() throws Exception {
         indexRandom(false, false, false, Arrays.asList(builders));  // this one
 
         // Corrupt the translog file(s)
-        corruptRandomTranslogFiles();
+        corruptRandomTranslogFile();
 
         // Restart the single node
         internalCluster().fullRestart();
@@ -102,7 +103,7 @@ public void testCorruptTranslogFiles() throws Exception {
     }
 
 
-    private void corruptRandomTranslogFiles() throws IOException {
+    private void corruptRandomTranslogFile() throws IOException {
         ClusterState state = client().admin().cluster().prepareState().get().getState();
         GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
         final Index test = state.metaData().index("test").getIndex();
@@ -119,9 +120,12 @@ private void corruptRandomTranslogFiles() throws IOException {
             String path = fsPath.getPath();
             String relativeDataLocationPath = "indices/" + test.getUUID() + "/" + Integer.toString(shardRouting.getId()) + "/translog";
             Path translogDir = PathUtils.get(path).resolve(relativeDataLocationPath);
-            translogDirs.add(translogDir);
+            if (Files.isDirectory(translogDir)) {
+                translogDirs.add(translogDir);
+            }
         }
-        TestTranslog.corruptTranslogFiles(logger, random(), translogDirs);
+        Path translogDir = RandomPicks.randomFrom(random(), translogDirs);
+        TestTranslog.corruptRandomTranslogFile(logger, random(), translogDir, TestTranslog.minTranslogGenUsedInRecovery(translogDir));
     }
 
     /** Disables translog flushing for the specified index */
diff --git a/server/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/server/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
index 86519acf0926a..017fb360de58a 100644
--- a/server/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
+++ b/server/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
@@ -38,8 +38,6 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
@@ -95,18 +93,14 @@ public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, I
         for (NodeStats dataNode : nodeStats.getNodes()) {
             MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class,
                 dataNode.getNode().getName()));
-            mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()),
-                new MockTransportService.DelegateTransport(mockTransportService.original()) {
-                @Override
-                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
-                                           TransportRequestOptions options) throws IOException {
-                    super.sendRequest(connection, requestId, action, request, options);
+            mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()),
+                (connection, requestId, action, request, options) -> {
+                    connection.sendRequest(requestId, action, request, options);
                     if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
                         logger.debug("Throw ConnectTransportException");
                         throw new ConnectTransportException(connection.getNode(), action);
                     }
-                }
-            });
+                });
         }
 
         BulkRequestBuilder bulkBuilder = client.prepareBulk();
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java b/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java
index 7ab9fa6733011..f37ec5a8e55d5 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java
@@ -34,8 +34,6 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
-import java.util.Collection;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
 import java.util.Set;
@@ -45,7 +43,8 @@
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.not;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
 
 /**
  * Helpers for testing translog.
@@ -54,44 +53,33 @@ public class TestTranslog {
     static final Pattern TRANSLOG_FILE_PATTERN = Pattern.compile("translog-(\\d+)\\.tlog");
 
     /**
-     * Corrupts some translog files (translog-N.tlog) from the given translog directories.
+     * Corrupts random translog file (translog-N.tlog) from the given translog directory.
      *
-     * @return a collection of tlog files that have been corrupted.
+     * @return a translog file which has been corrupted.
      */
-    public static Set<Path> corruptTranslogFiles(Logger logger, Random random, Collection<Path> translogDirs) throws IOException {
+    public static Path corruptRandomTranslogFile(Logger logger, Random random, Path translogDir, long minGeneration) throws
+            IOException {
         Set<Path> candidates = new TreeSet<>(); // TreeSet makes sure iteration order is deterministic
-        for (Path translogDir : translogDirs) {
-            if (Files.isDirectory(translogDir)) {
-                final long minUsedTranslogGen = minTranslogGenUsedInRecovery(translogDir);
-                logger.info("--> Translog dir [{}], minUsedTranslogGen [{}]", translogDir, minUsedTranslogGen);
-                try (DirectoryStream<Path> stream = Files.newDirectoryStream(translogDir)) {
-                    for (Path item : stream) {
-                        if (Files.isRegularFile(item)) {
-                            // Makes sure that we will corrupt tlog files that are referenced by the Checkpoint.
-                            final Matcher matcher = TRANSLOG_FILE_PATTERN.matcher(item.getFileName().toString());
-                            if (matcher.matches() && Long.parseLong(matcher.group(1)) >= minUsedTranslogGen) {
-                                candidates.add(item);
-                            }
-                        }
+        logger.info("--> Translog dir [{}], minUsedTranslogGen [{}]", translogDir, minGeneration);
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(translogDir)) {
+            for (Path item : stream) {
+                if (Files.isRegularFile(item)) {
+                    final Matcher matcher = TRANSLOG_FILE_PATTERN.matcher(item.getFileName().toString());
+                    if (matcher.matches() && Long.parseLong(matcher.group(1)) >= minGeneration) {
+                        candidates.add(item);
                     }
                 }
             }
         }
+        assertThat(candidates, is(not(empty())));
 
-        Set<Path> corruptedFiles = new HashSet<>();
-        if (!candidates.isEmpty()) {
-            int corruptions = RandomNumbers.randomIntBetween(random, 5, 20);
-            for (int i = 0; i < corruptions; i++) {
-                Path fileToCorrupt = RandomPicks.randomFrom(random, candidates);
-                corruptFile(logger, random, fileToCorrupt);
-                corruptedFiles.add(fileToCorrupt);
-            }
-        }
-        assertThat("no translog file corrupted", corruptedFiles, not(empty()));
-        return corruptedFiles;
+        Path corruptedFile = RandomPicks.randomFrom(random, candidates);
+        corruptFile(logger, random, corruptedFile);
+        return corruptedFile;
     }
 
-    static void corruptFile(Logger logger, Random random, Path fileToCorrupt) throws IOException {
+
+     static void corruptFile(Logger logger, Random random, Path fileToCorrupt) throws IOException {
         try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
             // read
             raf.position(RandomNumbers.randomLongBetween(random, 0, raf.size() - 1));
@@ -117,7 +105,7 @@ static void corruptFile(Logger logger, Random random, Path fileToCorrupt) throws
     /**
      * Lists all existing commits in a given index path, then read the minimum translog generation that will be used in recoverFromTranslog.
      */
-    private static long minTranslogGenUsedInRecovery(Path translogPath) throws IOException {
+    public static long minTranslogGenUsedInRecovery(Path translogPath) throws IOException {
         try (NIOFSDirectory directory = new NIOFSDirectory(translogPath.getParent().resolve("index"))) {
             List<IndexCommit> commits = DirectoryReader.listCommits(directory);
             final String translogUUID = commits.get(commits.size() - 1).getUserData().get(Translog.TRANSLOG_UUID_KEY);
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index b255238c8648c..1c27a59e0ecbe 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -748,7 +748,9 @@ public void testConcurrentWritesWithVaryingSize() throws Throwable {
 
     }
 
-    public void testTranslogChecksums() throws Exception {
+    public void testTranslogCorruption() throws Exception {
+        TranslogConfig config = translog.getConfig();
+        String uuid = translog.getTranslogUUID();
         List<Translog.Location> locations = new ArrayList<>();
 
         int translogOperations = randomIntBetween(10, 100);
@@ -756,23 +758,23 @@ public void testTranslogChecksums() throws Exception {
             String ascii = randomAlphaOfLengthBetween(1, 50);
             locations.add(translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), ascii.getBytes("UTF-8"))));
         }
-        translog.sync();
+        translog.close();
 
-        corruptTranslogs(translogDir);
+        TestTranslog.corruptRandomTranslogFile(logger, random(), translogDir, 0);
+        int corruptionsCaught = 0;
 
-        AtomicInteger corruptionsCaught = new AtomicInteger(0);
-        try (Translog.Snapshot snapshot = translog.newSnapshot()) {
-            for (Translog.Location location : locations) {
-                try {
-                    Translog.Operation next = snapshot.next();
-                    assertNotNull(next);
-                } catch (TranslogCorruptedException e) {
-                    corruptionsCaught.incrementAndGet();
+        try (Translog translog = openTranslog(config, uuid)) {
+            try (Translog.Snapshot snapshot = translog.newSnapshot()) {
+                for (Location loc : locations) {
+                    snapshot.next();
                 }
             }
-            expectThrows(TranslogCorruptedException.class, snapshot::next);
-            assertThat("at least one corruption was caused and caught", corruptionsCaught.get(), greaterThanOrEqualTo(1));
+        } catch (TranslogCorruptedException e) {
+            assertThat(e.getMessage(), containsString(translogDir.toString()));
+            corruptionsCaught++;
         }
+
+        assertThat("corruption is caught", corruptionsCaught, greaterThanOrEqualTo(1));
     }
 
     public void testTruncatedTranslogs() throws Exception {
@@ -816,25 +818,6 @@ private void truncateTranslogs(Path directory) throws Exception {
     }
 
 
-    /**
-     * Randomly overwrite some bytes in the translog files
-     */
-    private void corruptTranslogs(Path directory) throws Exception {
-        Path[] files = FileSystemUtils.files(directory, "translog-*");
-        for (Path file : files) {
-            logger.info("--> corrupting {}...", file);
-            FileChannel f = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE);
-            int corruptions = scaledRandomIntBetween(10, 50);
-            for (int i = 0; i < corruptions; i++) {
-                // note: with the current logic, this will sometimes be a no-op
-                long pos = randomIntBetween(0, (int) f.size());
-                ByteBuffer junk = ByteBuffer.wrap(new byte[]{randomByte()});
-                f.write(junk, pos);
-            }
-            f.close();
-        }
-    }
-
     private Term newUid(ParsedDocument doc) {
         return new Term("_id", Uid.encodeId(doc.id()));
     }
@@ -1505,7 +1488,8 @@ public void testSnapshotFromStreamInput() throws IOException {
             ops.add(test);
         }
         Translog.writeOperations(out, ops);
-        final List<Translog.Operation> readOperations = Translog.readOperations(out.bytes().streamInput());
+        final List<Translog.Operation> readOperations = Translog.readOperations(
+                out.bytes().streamInput(), "testSnapshotFromStreamInput");
         assertEquals(ops.size(), readOperations.size());
         assertEquals(ops, readOperations);
     }
@@ -2669,7 +2653,7 @@ public void testTranslogOpSerialization() throws Exception {
 
         Engine.Index eIndex = new Engine.Index(newUid(doc), doc, randomSeqNum, randomPrimaryTerm,
             1, VersionType.INTERNAL, Origin.PRIMARY, 0, 0, false);
-        Engine.IndexResult eIndexResult = new Engine.IndexResult(1, randomSeqNum, true);
+        Engine.IndexResult eIndexResult = new Engine.IndexResult(1, randomPrimaryTerm, randomSeqNum, true);
         Translog.Index index = new Translog.Index(eIndex, eIndexResult);
 
         BytesStreamOutput out = new BytesStreamOutput();
@@ -2680,7 +2664,7 @@ public void testTranslogOpSerialization() throws Exception {
 
         Engine.Delete eDelete = new Engine.Delete(doc.type(), doc.id(), newUid(doc), randomSeqNum, randomPrimaryTerm,
             2, VersionType.INTERNAL, Origin.PRIMARY, 0);
-        Engine.DeleteResult eDeleteResult = new Engine.DeleteResult(2, randomSeqNum, true);
+        Engine.DeleteResult eDeleteResult = new Engine.DeleteResult(2, randomPrimaryTerm, randomSeqNum, true);
         Translog.Delete delete = new Translog.Delete(eDelete, eDeleteResult);
 
         out = new BytesStreamOutput();
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java b/server/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
index 029ed50fb2851..cd4605b7e2d01 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
@@ -153,9 +153,9 @@ public void testCorruptTranslogTruncation() throws Exception {
         // shut down the replica node to be tested later
         internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode));
 
-        // Corrupt the translog file(s)
+        // Corrupt the translog file
         logger.info("--> corrupting translog");
-        corruptRandomTranslogFiles("test");
+        corruptRandomTranslogFile("test");
 
         // Restart the single node
         logger.info("--> restarting node");
@@ -267,15 +267,16 @@ public void testCorruptTranslogTruncationOfReplica() throws Exception {
         // sample the replica node translog dirs
         final ShardId shardId = new ShardId(resolveIndex("test"), 0);
         Set<Path> translogDirs = getTranslogDirs(replicaNode, shardId);
+        Path tdir = randomFrom(translogDirs);
 
         // stop the cluster nodes. we don't use full restart so the node start up order will be the same
         // and shard roles will be maintained
         internalCluster().stopRandomDataNode();
         internalCluster().stopRandomDataNode();
 
-        // Corrupt the translog file(s)
+        // Corrupt the translog file
         logger.info("--> corrupting translog");
-        TestTranslog.corruptTranslogFiles(logger, random(), translogDirs);
+        TestTranslog.corruptRandomTranslogFile(logger, random(), tdir, TestTranslog.minTranslogGenUsedInRecovery(tdir));
 
         // Restart the single node
         logger.info("--> starting node");
@@ -358,9 +359,10 @@ private Set<Path> getTranslogDirs(String nodeId, ShardId shardId) {
         return translogDirs;
     }
 
-    private void corruptRandomTranslogFiles(String indexName) throws IOException {
+    private void corruptRandomTranslogFile(String indexName) throws IOException {
         Set<Path> translogDirs = getTranslogDirs(indexName);
-        TestTranslog.corruptTranslogFiles(logger, random(), translogDirs);
+        Path translogDir = randomFrom(translogDirs);
+        TestTranslog.corruptRandomTranslogFile(logger, random(), translogDir, TestTranslog.minTranslogGenUsedInRecovery(translogDir));
     }
 
     /** Disables translog flushing for the specified index */
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
index 5c6b000f7e519..580696264bdd4 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
@@ -74,6 +74,10 @@ public void injectRandomFailures() {
         enableRandomFailures = randomBoolean();
     }
 
+    protected void disableRandomFailures() {
+        enableRandomFailures = false;
+    }
+
     protected void failRandomly() {
         if (enableRandomFailures && rarely()) {
             throw new RuntimeException("dummy test failure");
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index 5611421594aa1..39091ce04ec6e 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -75,6 +76,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -97,7 +99,6 @@ public void tearDown() throws Exception {
         terminate(threadPool);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32308")
     public void testRandomClusterStateUpdates() {
         // we have an IndicesClusterStateService per node in the cluster
         final Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap = new HashMap<>();
@@ -199,6 +200,59 @@ public void testJoiningNewClusterOnlyRemovesInMemoryIndexStructures() {
         }
     }
 
+    /**
+     * In rare cases it is possible that a nodes gets an instruction to replace a replica
+     * shard that's in POST_RECOVERY with a new initializing primary with the same allocation id.
+     * This can happen by batching cluster states that include the starting of the replica, with
+     * closing of the indices, opening it up again and allocating the primary shard to the node in
+     * question. The node should then clean it's initializing replica and replace it with a new
+     * initializing primary.
+     */
+    public void testInitializingPrimaryRemovesInitializingReplicaWithSameAID() {
+        disableRandomFailures();
+        String index = "index_" + randomAlphaOfLength(8).toLowerCase(Locale.ROOT);
+        ClusterState state = ClusterStateCreationUtils.state(index, randomBoolean(),
+            ShardRoutingState.STARTED, ShardRoutingState.INITIALIZING);
+
+        // the initial state which is derived from the newly created cluster state but doesn't contain the index
+        ClusterState previousState = ClusterState.builder(state)
+            .metaData(MetaData.builder(state.metaData()).remove(index))
+            .routingTable(RoutingTable.builder().build())
+            .build();
+
+        // pick a data node to simulate the adding an index cluster state change event on, that has shards assigned to it
+        final ShardRouting shardRouting = state.routingTable().index(index).shard(0).replicaShards().get(0);
+        final ShardId shardId = shardRouting.shardId();
+        DiscoveryNode node = state.nodes().get(shardRouting.currentNodeId());
+
+        // simulate the cluster state change on the node
+        ClusterState localState = adaptClusterStateToLocalNode(state, node);
+        ClusterState previousLocalState = adaptClusterStateToLocalNode(previousState, node);
+        IndicesClusterStateService indicesCSSvc = createIndicesClusterStateService(node, RecordingIndicesService::new);
+        indicesCSSvc.start();
+        indicesCSSvc.applyClusterState(new ClusterChangedEvent("cluster state change that adds the index", localState, previousLocalState));
+        previousState = state;
+
+        // start the replica
+        state = cluster.applyStartedShards(state, state.routingTable().index(index).shard(0).replicaShards());
+
+        // close the index and open it up again (this will sometimes swap roles between primary and replica)
+        CloseIndexRequest closeIndexRequest = new CloseIndexRequest(state.metaData().index(index).getIndex().getName());
+        state = cluster.closeIndices(state, closeIndexRequest);
+        OpenIndexRequest openIndexRequest = new OpenIndexRequest(state.metaData().index(index).getIndex().getName());
+        state = cluster.openIndices(state, openIndexRequest);
+
+        localState = adaptClusterStateToLocalNode(state, node);
+        previousLocalState = adaptClusterStateToLocalNode(previousState, node);
+
+        indicesCSSvc.applyClusterState(new ClusterChangedEvent("new cluster state", localState, previousLocalState));
+
+        final MockIndexShard shardOrNull = ((RecordingIndicesService) indicesCSSvc.indicesService).getShardOrNull(shardId);
+        assertThat(shardOrNull == null ? null : shardOrNull.routingEntry(),
+            equalTo(state.getRoutingNodes().node(node.getId()).getByShardId(shardId)));
+
+    }
+
     public ClusterState randomInitialClusterState(Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap,
                                                   Supplier<MockIndicesService> indicesServiceSupplier) {
         List<DiscoveryNode> allNodes = new ArrayList<>();
diff --git a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
index bdd01e88ccad4..4483ce0d60642 100644
--- a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
@@ -286,7 +286,6 @@ public void testSyncedFlushSkipOutOfSyncReplicas() throws Exception {
         assertThat(fullResult.successfulShards(), equalTo(numberOfReplicas + 1));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32436")
     public void testDoNotRenewSyncedFlushWhenAllSealed() throws Exception {
         internalCluster().ensureAtLeastNumDataNodes(between(2, 3));
         final int numberOfReplicas = internalCluster().numDataNodes() - 1;
@@ -311,7 +310,7 @@ public void testDoNotRenewSyncedFlushWhenAllSealed() throws Exception {
         // Shards were updated, renew synced flush.
         final int moreDocs = between(1, 10);
         for (int i = 0; i < moreDocs; i++) {
-            index("test", "doc", Integer.toString(i));
+            index("test", "doc", "more-" + i);
         }
         final ShardsSyncedFlushResult thirdSeal = SyncedFlushUtil.attemptSyncedFlush(logger, internalCluster(), shardId);
         assertThat(thirdSeal.successfulShards(), equalTo(numberOfReplicas + 1));
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
index cf1449fecd6a5..89a8813e3e07b 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
@@ -61,6 +61,7 @@
 import org.elasticsearch.test.store.MockFSDirectoryService;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.test.transport.StubbableTransport;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportRequest;
@@ -598,8 +599,8 @@ public void testDisconnectsWhileRecovering() throws Exception {
         TransportService blueTransportService = internalCluster().getInstance(TransportService.class, blueNodeName);
         final CountDownLatch requestBlocked = new CountDownLatch(1);
 
-        blueMockTransportService.addDelegate(redTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, blueMockTransportService.original(), requestBlocked));
-        redMockTransportService.addDelegate(blueTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, redMockTransportService.original(), requestBlocked));
+        blueMockTransportService.addSendBehavior(redTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, requestBlocked));
+        redMockTransportService.addSendBehavior(blueTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, requestBlocked));
 
         logger.info("--> starting recovery from blue to red");
         client().admin().indices().prepareUpdateSettings(indexName).setSettings(
@@ -620,21 +621,20 @@ public void testDisconnectsWhileRecovering() throws Exception {
 
     }
 
-    private class RecoveryActionBlocker extends MockTransportService.DelegateTransport {
+    private class RecoveryActionBlocker implements StubbableTransport.SendRequestBehavior {
         private final boolean dropRequests;
         private final String recoveryActionToBlock;
         private final CountDownLatch requestBlocked;
 
-        RecoveryActionBlocker(boolean dropRequests, String recoveryActionToBlock, Transport delegate, CountDownLatch requestBlocked) {
-            super(delegate);
+        RecoveryActionBlocker(boolean dropRequests, String recoveryActionToBlock, CountDownLatch requestBlocked) {
             this.dropRequests = dropRequests;
             this.recoveryActionToBlock = recoveryActionToBlock;
             this.requestBlocked = requestBlocked;
         }
 
         @Override
-        protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
-                                   TransportRequestOptions options) throws IOException {
+        public void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request,
+                                TransportRequestOptions options) throws IOException {
             if (recoveryActionToBlock.equals(action) || requestBlocked.getCount() == 0) {
                 logger.info("--> preventing {} request", action);
                 requestBlocked.countDown();
@@ -643,7 +643,7 @@ protected void sendRequest(Connection connection, long requestId, String action,
                 }
                 throw new ConnectTransportException(connection.getNode(), "DISCONNECT: prevented " + action + " request");
             }
-            super.sendRequest(connection, requestId, action, request, options);
+            connection.sendRequest(requestId, action, request, options);
         }
     }
 
@@ -686,12 +686,12 @@ public void testDisconnectsDuringRecovery() throws Exception {
         MockTransportService blueMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, blueNodeName);
         MockTransportService redMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, redNodeName);
 
-        redMockTransportService.addDelegate(blueMockTransportService, new MockTransportService.DelegateTransport(redMockTransportService.original()) {
+        redMockTransportService.addSendBehavior(blueMockTransportService, new StubbableTransport.SendRequestBehavior() {
             private final AtomicInteger count = new AtomicInteger();
 
             @Override
-            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
-                                       TransportRequestOptions options) throws IOException {
+            public void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request,
+                                    TransportRequestOptions options) throws IOException {
                 logger.info("--> sending request {} on {}", action, connection.getNode());
                 if (PeerRecoverySourceService.Actions.START_RECOVERY.equals(action) && count.incrementAndGet() == 1) {
                     // ensures that it's considered as valid recovery attempt by source
@@ -701,7 +701,7 @@ protected void sendRequest(Connection connection, long requestId, String action,
                     } catch (InterruptedException e) {
                         throw new RuntimeException(e);
                     }
-                    super.sendRequest(connection, requestId, action, request, options);
+                    connection.sendRequest(requestId, action, request, options);
                     try {
                         Thread.sleep(disconnectAfterDelay.millis());
                     } catch (InterruptedException e) {
@@ -709,35 +709,27 @@ protected void sendRequest(Connection connection, long requestId, String action,
                     }
                     throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulation disconnect after successfully sending " + action + " request");
                 } else {
-                    super.sendRequest(connection, requestId, action, request, options);
+                    connection.sendRequest(requestId, action, request, options);
                 }
             }
         });
 
         final AtomicBoolean finalized = new AtomicBoolean();
-        blueMockTransportService.addDelegate(redMockTransportService, new MockTransportService.DelegateTransport(blueMockTransportService.original()) {
-            @Override
-            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
-                                       TransportRequestOptions options) throws IOException {
-                logger.info("--> sending request {} on {}", action, connection.getNode());
-                if (action.equals(PeerRecoveryTargetService.Actions.FINALIZE)) {
-                    finalized.set(true);
-                }
-                super.sendRequest(connection, requestId, action, request, options);
+        blueMockTransportService.addSendBehavior(redMockTransportService, (connection, requestId, action, request, options) -> {
+            logger.info("--> sending request {} on {}", action, connection.getNode());
+            if (action.equals(PeerRecoveryTargetService.Actions.FINALIZE)) {
+                finalized.set(true);
             }
+            connection.sendRequest(requestId, action, request, options);
         });
 
         for (MockTransportService mockTransportService : Arrays.asList(redMockTransportService, blueMockTransportService)) {
-            mockTransportService.addDelegate(masterTransportService, new MockTransportService.DelegateTransport(mockTransportService.original()) {
-                @Override
-                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
-                                           TransportRequestOptions options) throws IOException {
-                    logger.info("--> sending request {} on {}", action, connection.getNode());
-                    if ((primaryRelocation && finalized.get()) == false) {
-                        assertNotEquals(action, ShardStateAction.SHARD_FAILED_ACTION_NAME);
-                    }
-                    super.sendRequest(connection, requestId, action, request, options);
+            mockTransportService.addSendBehavior(masterTransportService, (connection, requestId, action, request, options) -> {
+                logger.info("--> sending request {} on {}", action, connection.getNode());
+                if ((primaryRelocation && finalized.get()) == false) {
+                    assertNotEquals(action, ShardStateAction.SHARD_FAILED_ACTION_NAME);
                 }
+                connection.sendRequest(requestId, action, request, options);
             });
         }
 
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
index 6be6d7e80bccb..f0644b029c3dc 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
@@ -179,12 +179,12 @@ public void testSendSnapshotSendsOps() throws IOException {
         final int initialNumberOfDocs = randomIntBetween(16, 64);
         for (int i = 0; i < initialNumberOfDocs; i++) {
             final Engine.Index index = getIndex(Integer.toString(i));
-            operations.add(new Translog.Index(index, new Engine.IndexResult(1, SequenceNumbers.UNASSIGNED_SEQ_NO, true)));
+            operations.add(new Translog.Index(index, new Engine.IndexResult(1, 1, SequenceNumbers.UNASSIGNED_SEQ_NO, true)));
         }
         final int numberOfDocsWithValidSequenceNumbers = randomIntBetween(16, 64);
         for (int i = initialNumberOfDocs; i < initialNumberOfDocs + numberOfDocsWithValidSequenceNumbers; i++) {
             final Engine.Index index = getIndex(Integer.toString(i));
-            operations.add(new Translog.Index(index, new Engine.IndexResult(1, i - initialNumberOfDocs, true)));
+            operations.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i - initialNumberOfDocs, true)));
         }
         operations.add(null);
         final long startingSeqNo = randomIntBetween(0, numberOfDocsWithValidSequenceNumbers - 1);
@@ -395,7 +395,7 @@ public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Started() throws IOE
         final IndexShard shard = mock(IndexShard.class);
         when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class));
         when(shard.segmentStats(anyBoolean())).thenReturn(mock(SegmentsStats.class));
-        when(shard.isPrimaryMode()).thenReturn(false);
+        when(shard.isRelocatedPrimary()).thenReturn(true);
         when(shard.acquireSafeIndexCommit()).thenReturn(mock(Engine.IndexCommitRef.class));
         doAnswer(invocation -> {
             ((ActionListener<Releasable>)invocation.getArguments()[0]).onResponse(() -> {});
@@ -444,7 +444,7 @@ public void testCancellationsDoesNotLeakPrimaryPermits() throws Exception {
         final CancellableThreads cancellableThreads = new CancellableThreads();
         final IndexShard shard = mock(IndexShard.class);
         final AtomicBoolean freed = new AtomicBoolean(true);
-        when(shard.isPrimaryMode()).thenReturn(true);
+        when(shard.isRelocatedPrimary()).thenReturn(false);
         doAnswer(invocation -> {
             freed.set(false);
             ((ActionListener<Releasable>)invocation.getArguments()[0]).onResponse(() -> freed.set(true));
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
index aaba17c315187..7ca9dcc82f70a 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
@@ -43,6 +43,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.translog.SnapshotMatchers;
 import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.HashMap;
 import java.util.List;
@@ -73,6 +74,7 @@ public void testTranslogHistoryTransferred() throws Exception {
         }
     }
 
+    @TestLogging("_root:TRACE")
     public void testRetentionPolicyChangeDuringRecovery() throws Exception {
         try (ReplicationGroup shards = createGroup(0)) {
             shards.startPrimary();
@@ -99,7 +101,10 @@ public void testRetentionPolicyChangeDuringRecovery() throws Exception {
             releaseRecovery.countDown();
             future.get();
             // rolling/flushing is async
-            assertBusy(() -> assertThat(replica.estimateTranslogOperationsFromMinSeq(0), equalTo(0)));
+            assertBusy(() -> {
+                assertThat(replica.getLastSyncedGlobalCheckpoint(), equalTo(19L));
+                assertThat(replica.estimateTranslogOperationsFromMinSeq(0), equalTo(0));
+            });
         }
     }
 
@@ -201,7 +206,7 @@ public void testDifferentHistoryUUIDDisablesOPsRecovery() throws Exception {
             if (randomBoolean()) {
                 // create a new translog
                 translogUUIDtoUse = Translog.createEmptyTranslog(replica.shardPath().resolveTranslog(), flushedDocs,
-                    replica.shardId(), replica.getPrimaryTerm());
+                    replica.shardId(), replica.getPendingPrimaryTerm());
                 translogGenToUse = 1;
             } else {
                 translogUUIDtoUse = translogGeneration.translogUUID;
diff --git a/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index 4d0d0d6ff3fd9..5ed4b37030786 100644
--- a/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -1004,6 +1004,7 @@ private void assertCumulativeQueryCacheStats(IndicesStatsResponse response) {
         assertEquals(total, shardTotal);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32506")
     public void testFilterCacheStats() throws Exception {
         assertAcked(prepareCreate("index").setSettings(Settings.builder().put(indexSettings()).put("number_of_replicas", 0).build()).get());
         indexRandom(true,
diff --git a/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
index 5c1c0d91e608b..7e99ccbbe6117 100644
--- a/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
@@ -55,8 +55,6 @@
 import org.elasticsearch.test.disruption.BlockClusterStateProcessing;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
@@ -234,16 +232,13 @@ public void testShardCleanupIfShardDeletionAfterRelocationFailedAndIndexDeleted(
         MockTransportService transportServiceNode_1 = (MockTransportService) internalCluster().getInstance(TransportService.class, node_1);
         TransportService transportServiceNode_2 = internalCluster().getInstance(TransportService.class, node_2);
         final CountDownLatch shardActiveRequestSent = new CountDownLatch(1);
-        transportServiceNode_1.addDelegate(transportServiceNode_2, new MockTransportService.DelegateTransport(transportServiceNode_1.original()) {
-            @Override
-            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
-                if (action.equals("internal:index/shard/exists") && shardActiveRequestSent.getCount() > 0) {
-                    shardActiveRequestSent.countDown();
-                    logger.info("prevent shard active request from being sent");
-                    throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulated");
-                }
-                super.sendRequest(connection, requestId, action, request, options);
+        transportServiceNode_1.addSendBehavior(transportServiceNode_2, (connection, requestId, action, request, options) -> {
+            if (action.equals("internal:index/shard/exists") && shardActiveRequestSent.getCount() > 0) {
+                shardActiveRequestSent.countDown();
+                logger.info("prevent shard active request from being sent");
+                throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulated");
             }
+            connection.sendRequest(requestId, action, request, options);
         });
 
         logger.info("--> move shard from {} to {}, and wait for relocation to finish", node_1, node_2);
diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
index 0481385909fe2..65139109a83a2 100644
--- a/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
+++ b/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
@@ -34,7 +34,7 @@
 import org.elasticsearch.action.ingest.SimulateDocumentBaseResult;
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -126,7 +126,7 @@ public void testSimulate() throws Exception {
         assertThat(simulateDocumentBaseResult.getFailure(), nullValue());
 
         // cleanup
-        WritePipelineResponse deletePipelineResponse = client().admin().cluster().prepareDeletePipeline("_id").get();
+        AcknowledgedResponse deletePipelineResponse = client().admin().cluster().prepareDeletePipeline("_id").get();
         assertTrue(deletePipelineResponse.isAcknowledged());
     }
 
@@ -172,7 +172,7 @@ public void testBulkWithIngestFailures() throws Exception {
         }
 
         // cleanup
-        WritePipelineResponse deletePipelineResponse = client().admin().cluster().prepareDeletePipeline("_id").get();
+        AcknowledgedResponse deletePipelineResponse = client().admin().cluster().prepareDeletePipeline("_id").get();
         assertTrue(deletePipelineResponse.isAcknowledged());
     }
 
@@ -246,7 +246,7 @@ public void test() throws Exception {
         assertThat(doc.get("processed"), equalTo(true));
 
         DeletePipelineRequest deletePipelineRequest = new DeletePipelineRequest("_id");
-        WritePipelineResponse response = client().admin().cluster().deletePipeline(deletePipelineRequest).get();
+        AcknowledgedResponse response = client().admin().cluster().deletePipeline(deletePipelineRequest).get();
         assertThat(response.isAcknowledged(), is(true));
 
         getResponse = client().admin().cluster().prepareGetPipeline("_id").get();
diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/server/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
index 84d9327a0910a..338e5b662c5da 100644
--- a/server/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
+++ b/server/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.node.NodeService;
@@ -95,7 +95,7 @@ public void testFailStartNode() throws Exception {
         installPlugin = true;
         String node1 = internalCluster().startNode();
 
-        WritePipelineResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get();
+        AcknowledgedResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get();
         assertThat(response.isAcknowledged(), is(true));
         Pipeline pipeline = internalCluster().getInstance(NodeService.class, node1).getIngestService().getPipelineStore().get("_id");
         assertThat(pipeline, notNullValue());
diff --git a/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java b/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java
index d86d7b46cc7a1..8aa0f3ec5bad6 100644
--- a/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java
@@ -34,7 +34,6 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 
@@ -53,7 +52,7 @@ public void setUp() throws Exception {
         threadpool = new TestThreadPool("response_collector_tests");
         clusterService = new ClusterService(Settings.EMPTY,
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                threadpool, Collections.emptyMap());
+                threadpool);
         collector = new ResponseCollectorService(Settings.EMPTY, clusterService);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index 738cd00a43d2e..cb93d803bb7c6 100644
--- a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -58,6 +58,7 @@
 import org.elasticsearch.test.MockIndexEventListener;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.test.transport.StubbableTransport;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
@@ -372,7 +373,7 @@ public void testCancellationCleansTempFiles() throws Exception {
         MockTransportService mockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, p_node);
         for (DiscoveryNode node : clusterService.state().nodes()) {
             if (!node.equals(clusterService.localNode())) {
-                mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, node.getName()), new RecoveryCorruption(mockTransportService.original(), corruptionCount));
+                mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, node.getName()), new RecoveryCorruption(corruptionCount));
             }
         }
 
@@ -485,17 +486,16 @@ public void testIndexAndRelocateConcurrently() throws ExecutionException, Interr
 
     }
 
-    class RecoveryCorruption extends MockTransportService.DelegateTransport {
+    class RecoveryCorruption implements StubbableTransport.SendRequestBehavior {
 
         private final CountDownLatch corruptionCount;
 
-        RecoveryCorruption(Transport transport, CountDownLatch corruptionCount) {
-            super(transport);
+        RecoveryCorruption(CountDownLatch corruptionCount) {
             this.corruptionCount = corruptionCount;
         }
 
         @Override
-        protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
+        public void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
             if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                 RecoveryFileChunkRequest chunkRequest = (RecoveryFileChunkRequest) request;
                 if (chunkRequest.name().startsWith(IndexFileNames.SEGMENTS)) {
@@ -506,9 +506,9 @@ protected void sendRequest(Connection connection, long requestId, String action,
                     array[0] = (byte) ~array[0]; // flip one byte in the content
                     corruptionCount.countDown();
                 }
-                super.sendRequest(connection, requestId, action, request, options);
+                connection.sendRequest(requestId, action, request, options);
             } else {
-                super.sendRequest(connection, requestId, action, request, options);
+                connection.sendRequest(requestId, action, request, options);
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java b/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
index b33cfb508b930..ac8688c9847d3 100644
--- a/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
+++ b/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
@@ -35,11 +35,8 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -70,8 +67,8 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
      */
     public void testCancelRecoveryAndResume() throws Exception {
         assertTrue(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
-                .put(CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(randomIntBetween(50, 300), ByteSizeUnit.BYTES)))
-                .get().isAcknowledged());
+            .put(CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(randomIntBetween(50, 300), ByteSizeUnit.BYTES)))
+            .get().isAcknowledged());
 
         NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
         List<NodeStats> dataNodeStats = new ArrayList<>();
@@ -91,9 +88,9 @@ public void testCancelRecoveryAndResume() throws Exception {
         // create the index and prevent allocation on any other nodes than the lucky one
         // we have no replicas so far and make sure that we allocate the primary on the lucky node
         assertAcked(prepareCreate("test")
-                .addMapping("type1", "field1", "type=text", "the_id", "type=text")
-                .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards())
-                        .put("index.routing.allocation.include._name", primariesNode.getNode().getName()))); // only allocate on the lucky node
+            .addMapping("type1", "field1", "type=text", "the_id", "type=text")
+            .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards())
+                .put("index.routing.allocation.include._name", primariesNode.getNode().getName()))); // only allocate on the lucky node
 
         // index some docs and check if they are coming back
         int numDocs = randomIntBetween(100, 200);
@@ -116,11 +113,8 @@ public void testCancelRecoveryAndResume() throws Exception {
         final AtomicBoolean truncate = new AtomicBoolean(true);
         for (NodeStats dataNode : dataNodeStats) {
             MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
-            mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {
-
-                @Override
-                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
-                                           TransportRequestOptions options) throws IOException {
+            mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()),
+                (connection, requestId, action, request, options) -> {
                     if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                         RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
                         logger.debug("file chunk [{}] lastChunk: {}", req, req.lastChunk());
@@ -129,16 +123,15 @@ protected void sendRequest(Connection connection, long requestId, String action,
                             throw new RuntimeException("Caused some truncated files for fun and profit");
                         }
                     }
-                    super.sendRequest(connection, requestId, action, request, options);
-                }
-            });
+                    connection.sendRequest(requestId, action, request, options);
+                });
         }
 
         logger.info("--> bumping replicas to 1"); //
         client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder()
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-                .put("index.routing.allocation.include._name",  // now allow allocation on all nodes
-                        primariesNode.getNode().getName() + "," + unluckyNode.getNode().getName())).get();
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put("index.routing.allocation.include._name",  // now allow allocation on all nodes
+                primariesNode.getNode().getName() + "," + unluckyNode.getNode().getName())).get();
 
         latch.await();
 
diff --git a/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java b/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java
index 401cc79b02092..d44b2b70a05ef 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java
@@ -20,22 +20,33 @@
 package org.elasticsearch.rest.action;
 
 import com.fasterxml.jackson.core.io.JsonEOFException;
-import java.util.Arrays;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 import static java.util.Collections.emptyList;
+import static org.hamcrest.CoreMatchers.equalTo;
 
 public class RestActionsTests extends ESTestCase {
 
@@ -98,9 +109,101 @@ public void testParseTopLevelBuilderUnknownParameter() throws IOException {
         }
     }
 
+    public void testBuildBroadcastShardsHeader() throws IOException {
+        ShardOperationFailedException[] failures = new ShardOperationFailedException[]{
+            createShardFailureParsingException("node0", 0, null),
+            createShardFailureParsingException("node1", 1, null),
+            createShardFailureParsingException("node2", 2, null),
+            createShardFailureParsingException("node0", 0, "cluster1"),
+            createShardFailureParsingException("node1", 1, "cluster1"),
+            createShardFailureParsingException("node2", 2, "cluster1"),
+            createShardFailureParsingException("node0", 0, "cluster2"),
+            createShardFailureParsingException("node1", 1, "cluster2"),
+            createShardFailureParsingException("node2", 2, "cluster2")
+        };
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.prettyPrint();
+        builder.startObject();
+        RestActions.buildBroadcastShardsHeader(builder, ToXContent.EMPTY_PARAMS, 12, 3, 0, 9, failures);
+        builder.endObject();
+        assertThat(Strings.toString(builder), equalTo("{\n" +
+            "  \"_shards\" : {\n" +
+            "    \"total\" : 12,\n" +
+            "    \"successful\" : 3,\n" +
+            "    \"skipped\" : 0,\n" +
+            "    \"failed\" : 9,\n" +
+            "    \"failures\" : [\n" +
+            "      {\n" +
+            "        \"shard\" : 0,\n" +
+            "        \"index\" : \"index\",\n" +
+            "        \"node\" : \"node0\",\n" +
+            "        \"reason\" : {\n" +
+            "          \"type\" : \"parsing_exception\",\n" +
+            "          \"reason\" : \"error\",\n" +
+            "          \"index_uuid\" : \"_na_\",\n" +
+            "          \"index\" : \"index\",\n" +
+            "          \"line\" : 0,\n" +
+            "          \"col\" : 0,\n" +
+            "          \"caused_by\" : {\n" +
+            "            \"type\" : \"illegal_argument_exception\",\n" +
+            "            \"reason\" : \"some bad argument\"\n" +
+            "          }\n" +
+            "        }\n" +
+            "      },\n" +
+            "      {\n" +
+            "        \"shard\" : 0,\n" +
+            "        \"index\" : \"cluster1:index\",\n" +
+            "        \"node\" : \"node0\",\n" +
+            "        \"reason\" : {\n" +
+            "          \"type\" : \"parsing_exception\",\n" +
+            "          \"reason\" : \"error\",\n" +
+            "          \"index_uuid\" : \"_na_\",\n" +
+            "          \"index\" : \"index\",\n" +
+            "          \"line\" : 0,\n" +
+            "          \"col\" : 0,\n" +
+            "          \"caused_by\" : {\n" +
+            "            \"type\" : \"illegal_argument_exception\",\n" +
+            "            \"reason\" : \"some bad argument\"\n" +
+            "          }\n" +
+            "        }\n" +
+            "      },\n" +
+            "      {\n" +
+            "        \"shard\" : 0,\n" +
+            "        \"index\" : \"cluster2:index\",\n" +
+            "        \"node\" : \"node0\",\n" +
+            "        \"reason\" : {\n" +
+            "          \"type\" : \"parsing_exception\",\n" +
+            "          \"reason\" : \"error\",\n" +
+            "          \"index_uuid\" : \"_na_\",\n" +
+            "          \"index\" : \"index\",\n" +
+            "          \"line\" : 0,\n" +
+            "          \"col\" : 0,\n" +
+            "          \"caused_by\" : {\n" +
+            "            \"type\" : \"illegal_argument_exception\",\n" +
+            "            \"reason\" : \"some bad argument\"\n" +
+            "          }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    ]\n" +
+            "  }\n" +
+            "}"));
+    }
+
+    private static ShardSearchFailure createShardFailureParsingException(String nodeId, int shardId, String clusterAlias) {
+        String index = "index";
+        ParsingException ex = new ParsingException(0, 0, "error", new IllegalArgumentException("some bad argument"));
+        ex.setIndex(index);
+        return new ShardSearchFailure(ex, createSearchShardTarget(nodeId, shardId, index, clusterAlias));
+    }
+
+    private static SearchShardTarget createSearchShardTarget(String nodeId, int shardId, String index, String clusterAlias) {
+        return new SearchShardTarget(nodeId,
+            new ShardId(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), clusterAlias, OriginalIndices.NONE);
+    }
+
     @Override
     protected NamedXContentRegistry xContentRegistry() {
         return xContentRegistry;
     }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 585f860165160..ea8b6a9223412 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -167,7 +167,7 @@ public void testAllowAllScriptContextSettings() throws IOException {
 
         assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.CONTEXT);
         assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.AGGS_CONTEXT);
-        assertCompileAccepted("painless", "script", ScriptType.INLINE, ExecutableScript.UPDATE_CONTEXT);
+        assertCompileAccepted("painless", "script", ScriptType.INLINE, UpdateScript.CONTEXT);
         assertCompileAccepted("painless", "script", ScriptType.INLINE, IngestScript.CONTEXT);
     }
 
@@ -187,7 +187,7 @@ public void testAllowSomeScriptContextSettings() throws IOException {
 
         assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.CONTEXT);
         assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.AGGS_CONTEXT);
-        assertCompileRejected("painless", "script", ScriptType.INLINE, ExecutableScript.UPDATE_CONTEXT);
+        assertCompileRejected("painless", "script", ScriptType.INLINE, UpdateScript.CONTEXT);
     }
 
     public void testAllowNoScriptTypeSettings() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
index 97dfad4645447..996faf03ffbf7 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
@@ -19,23 +19,31 @@
 
 package org.elasticsearch.search;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+
 import org.apache.lucene.search.Explanation;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.get.GetResultTests;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchHit.NestedIdentity;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
@@ -43,16 +51,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.RandomObjects;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Predicate;
-
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
 import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
@@ -63,8 +61,6 @@
 
 public class SearchHitTests extends ESTestCase {
 
-    private static Set<String> META_FIELDS = Sets.newHashSet("_uid", "_parent", "_routing", "_size", "_timestamp", "_ttl");
-
     public static SearchHit createTestItem(boolean withOptionalInnerHits) {
         int internalId = randomInt();
         String uid = randomAlphaOfLength(10);
@@ -75,18 +71,7 @@ public static SearchHit createTestItem(boolean withOptionalInnerHits) {
         }
         Map<String, DocumentField> fields = new HashMap<>();
         if (randomBoolean()) {
-            int size = randomIntBetween(0, 10);
-            for (int i = 0; i < size; i++) {
-                Tuple<List<Object>, List<Object>> values = RandomObjects.randomStoredFieldValues(random(),
-                        XContentType.JSON);
-                if (randomBoolean()) {
-                    String metaField = randomFrom(META_FIELDS);
-                    fields.put(metaField, new DocumentField(metaField, values.v1()));
-                } else {
-                    String fieldName = randomAlphaOfLengthBetween(5, 10);
-                    fields.put(fieldName, new DocumentField(fieldName, values.v1()));
-                }
-            }
+            fields = GetResultTests.randomDocumentFields(XContentType.JSON).v1();
         }
         SearchHit hit = new SearchHit(internalId, uid, type, nestedIdentity, fields);
         if (frequently()) {
@@ -109,7 +94,8 @@ public static SearchHit createTestItem(boolean withOptionalInnerHits) {
             int size = randomIntBetween(0, 5);
             Map<String, HighlightField> highlightFields = new HashMap<>(size);
             for (int i = 0; i < size; i++) {
-                highlightFields.put(randomAlphaOfLength(5), HighlightFieldTests.createTestItem());
+                HighlightField testItem = HighlightFieldTests.createTestItem();
+                highlightFields.put(testItem.getName(), testItem);
             }
             hit.highlightFields(highlightFields);
         }
@@ -133,9 +119,10 @@ public static SearchHit createTestItem(boolean withOptionalInnerHits) {
             hit.setInnerHits(innerHits);
         }
         if (randomBoolean()) {
+            String index = randomAlphaOfLengthBetween(5, 10);
+            String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10);
             hit.shard(new SearchShardTarget(randomAlphaOfLengthBetween(5, 10),
-                    new ShardId(new Index(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)), randomInt()), null,
-                    OriginalIndices.NONE));
+                new ShardId(new Index(index, randomAlphaOfLengthBetween(5, 10)), randomInt()), clusterAlias, OriginalIndices.NONE));
         }
         return hit;
     }
diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index c01fd7acd7bd5..8daef1d0f6c69 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.search;
 
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -64,8 +66,11 @@
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.rescore.RescorerBuilder;
-import org.elasticsearch.search.suggest.CustomSuggesterSearchIT.CustomSuggestionBuilder;
+import org.elasticsearch.search.suggest.Suggest.Suggestion;
+import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionSearchContext;
+import org.elasticsearch.search.suggest.term.TermSuggestion;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 
 import java.io.IOException;
@@ -98,7 +103,8 @@ public Map<String, Highlighter> getHighlighters() {
         SearchPlugin registersDupeSuggester = new SearchPlugin() {
             @Override
             public List<SearchPlugin.SuggesterSpec<?>> getSuggesters() {
-                return singletonList(new SuggesterSpec<>("term", TermSuggestionBuilder::new, TermSuggestionBuilder::fromXContent));
+                return singletonList(new SuggesterSpec<>(TermSuggestionBuilder.SUGGESTION_NAME,
+                    TermSuggestionBuilder::new, TermSuggestionBuilder::fromXContent, TermSuggestion::new));
             }
         };
         expectThrows(IllegalArgumentException.class, registryForPlugin(registersDupeSuggester));
@@ -183,9 +189,15 @@ public void testRegisterSuggester() {
         SearchModule module = new SearchModule(Settings.EMPTY, false, singletonList(new SearchPlugin() {
             @Override
             public List<SuggesterSpec<?>> getSuggesters() {
-                return singletonList(new SuggesterSpec<>("custom", CustomSuggestionBuilder::new, CustomSuggestionBuilder::fromXContent));
+                return singletonList(
+                    new SuggesterSpec<>(
+                        TestSuggestionBuilder.SUGGESTION_NAME,
+                        TestSuggestionBuilder::new,
+                        TestSuggestionBuilder::fromXContent,
+                        TestSuggestion::new));
             }
         }));
+
         assertEquals(1, module.getNamedXContents().stream()
                 .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) &&
                     e.name.match("term", LoggingDeprecationHandler.INSTANCE)).count());
@@ -197,7 +209,7 @@ public List<SuggesterSpec<?>> getSuggesters() {
                     e.name.match("completion", LoggingDeprecationHandler.INSTANCE)).count());
         assertEquals(1, module.getNamedXContents().stream()
                 .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) &&
-                    e.name.match("custom", LoggingDeprecationHandler.INSTANCE)).count());
+                    e.name.match("test", LoggingDeprecationHandler.INSTANCE)).count());
 
         assertEquals(1, module.getNamedWriteables().stream()
                 .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("term")).count());
@@ -206,7 +218,16 @@ public List<SuggesterSpec<?>> getSuggesters() {
         assertEquals(1, module.getNamedWriteables().stream()
                 .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("completion")).count());
         assertEquals(1, module.getNamedWriteables().stream()
-                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("custom")).count());
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("test")).count());
+
+        assertEquals(1, module.getNamedWriteables().stream()
+            .filter(e -> e.categoryClass.equals(Suggestion.class) && e.name.equals("term")).count());
+        assertEquals(1, module.getNamedWriteables().stream()
+            .filter(e -> e.categoryClass.equals(Suggestion.class) && e.name.equals("phrase")).count());
+        assertEquals(1, module.getNamedWriteables().stream()
+            .filter(e -> e.categoryClass.equals(Suggestion.class) && e.name.equals("completion")).count());
+        assertEquals(1, module.getNamedWriteables().stream()
+            .filter(e -> e.categoryClass.equals(Suggestion.class) && e.name.equals("test")).count());
     }
 
     public void testRegisterHighlighter() {
@@ -498,4 +519,77 @@ public RescoreContext innerBuildContext(int windowSize, QueryShardContext contex
             return null;
         }
     }
+
+    private static class TestSuggester extends Suggester<SuggestionSearchContext.SuggestionContext> {
+        @Override
+        protected Suggestion<? extends Suggestion.Entry<? extends Suggestion.Entry.Option>> innerExecute(
+                String name,
+                SuggestionSearchContext.SuggestionContext suggestion,
+                IndexSearcher searcher,
+                CharsRefBuilder spare) throws IOException {
+            return null;
+        }
+    }
+
+    private static class TestSuggestionBuilder extends SuggestionBuilder<TestSuggestionBuilder> {
+
+        public static final String SUGGESTION_NAME = "test";
+
+        TestSuggestionBuilder(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        protected void doWriteTo(StreamOutput out) throws IOException {}
+
+        public static TestSuggestionBuilder fromXContent(XContentParser parser) {
+            return null;
+        }
+
+        @Override
+        protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+            return null;
+        }
+
+        @Override
+        protected SuggestionSearchContext.SuggestionContext build(QueryShardContext context) throws IOException {
+            return null;
+        }
+
+        @Override
+        protected boolean doEquals(TestSuggestionBuilder other) {
+            return false;
+        }
+
+        @Override
+        protected int doHashCode() {
+            return 0;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return "test";
+        }
+    }
+
+    private static class TestSuggestion extends Suggestion {
+        TestSuggestion(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        protected Entry newEntry() {
+            return null;
+        }
+
+        @Override
+        protected Entry newEntry(StreamInput in) throws IOException {
+            return null;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return "test";
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index c58a158fc677d..2562683466a8c 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -345,6 +345,23 @@ searchSourceBuilder, new String[0], false, new AliasFilter(null, Strings.EMPTY_A
         }
     }
 
+    public void testIgnoreScriptfieldIfSizeZero() throws IOException {
+        createIndex("index");
+        final SearchService service = getInstanceFromNode(SearchService.class);
+        final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
+        final IndexShard indexShard = indexService.getShard(0);
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.scriptField("field" + 0,
+                new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
+        searchSourceBuilder.size(0);
+        try (SearchContext context = service.createContext(new ShardSearchLocalRequest(indexShard.shardId(), 1, SearchType.DEFAULT,
+                searchSourceBuilder, new String[0], false, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, true, null, null))) {
+                assertEquals(0, context.scriptFields().fields().size());
+        }
+    }
+
     public static class FailOnRewriteQueryPlugin extends Plugin implements SearchPlugin {
         @Override
         public List<QuerySpec<?>> getQueries() {
diff --git a/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java b/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java
index d1a9a15a3937c..d69039b72f56a 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java
@@ -46,13 +46,13 @@ public static SearchSortValues createTestItem() {
         List<Supplier<Object>> valueSuppliers = new ArrayList<>();
         // this should reflect all values that are allowed to go through the transport layer
         valueSuppliers.add(() -> null);
-        valueSuppliers.add(() -> randomInt());
-        valueSuppliers.add(() -> randomLong());
-        valueSuppliers.add(() -> randomDouble());
-        valueSuppliers.add(() -> randomFloat());
-        valueSuppliers.add(() -> randomByte());
-        valueSuppliers.add(() -> randomShort());
-        valueSuppliers.add(() -> randomBoolean());
+        valueSuppliers.add(ESTestCase::randomInt);
+        valueSuppliers.add(ESTestCase::randomLong);
+        valueSuppliers.add(ESTestCase::randomDouble);
+        valueSuppliers.add(ESTestCase::randomFloat);
+        valueSuppliers.add(ESTestCase::randomByte);
+        valueSuppliers.add(ESTestCase::randomShort);
+        valueSuppliers.add(ESTestCase::randomBoolean);
         valueSuppliers.add(() -> frequently() ? randomAlphaOfLengthBetween(1, 30) : randomRealisticUnicodeOfCodepointLength(30));
 
         int size = randomIntBetween(1, 20);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java
index 8d67941639fb4..cbc3424314ab3 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java
@@ -26,8 +26,8 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
-import org.elasticsearch.common.joda.FormatDateTimeFormatter;
-import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.time.CompoundDateTimeFormatter;
+import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.BucketOrder;
@@ -137,17 +137,20 @@ private static Document documentForDate(String field, long millis) {
     }
 
     public void testRewriteTimeZone() throws IOException {
-        FormatDateTimeFormatter format = Joda.forPattern("strict_date_optional_time");
+        CompoundDateTimeFormatter format = DateFormatters.forPattern("strict_date_optional_time");
 
         try (Directory dir = newDirectory();
                 IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
 
-            w.addDocument(documentForDate(DATE_FIELD_NAME, format.parser().parseDateTime("2018-03-11T11:55:00").getMillis()));
-            w.addDocument(documentForDate(DATE_FIELD_NAME, format.parser().parseDateTime("2017-10-30T18:13:00").getMillis()));
+            long millis1 = DateFormatters.toZonedDateTime(format.parse("2018-03-11T11:55:00")).toInstant().toEpochMilli();
+            w.addDocument(documentForDate(DATE_FIELD_NAME, millis1));
+            long millis2 = DateFormatters.toZonedDateTime(format.parse("2017-10-30T18:13:00")).toInstant().toEpochMilli();
+            w.addDocument(documentForDate(DATE_FIELD_NAME, millis2));
 
             try (IndexReader readerThatDoesntCross = DirectoryReader.open(w)) {
 
-                w.addDocument(documentForDate(DATE_FIELD_NAME, format.parser().parseDateTime("2018-03-25T02:44:00").getMillis()));
+                long millis3 = DateFormatters.toZonedDateTime(format.parse("2018-03-25T02:44:00")).toInstant().toEpochMilli();
+                w.addDocument(documentForDate(DATE_FIELD_NAME, millis3));
 
                 try (IndexReader readerThatCrosses = DirectoryReader.open(w)) {
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
index 96811ce424c91..981d263d7d633 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
@@ -20,8 +20,6 @@
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.rounding.DateTimeUnit;
-import org.elasticsearch.common.rounding.Rounding;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
@@ -51,14 +49,6 @@ public class InternalAutoDateHistogramTests extends InternalMultiBucketAggregati
     public void setUp() throws Exception {
         super.setUp();
         format = randomNumericDocValueFormat();
-
-        roundingInfos = new RoundingInfo[6];
-        roundingInfos[0] = new RoundingInfo(Rounding.builder(DateTimeUnit.SECOND_OF_MINUTE).build(), 1, 5, 10, 30);
-        roundingInfos[1] = new RoundingInfo(Rounding.builder(DateTimeUnit.MINUTES_OF_HOUR).build(), 1, 5, 10, 30);
-        roundingInfos[2] = new RoundingInfo(Rounding.builder(DateTimeUnit.HOUR_OF_DAY).build(), 1, 3, 12);
-        roundingInfos[3] = new RoundingInfo(Rounding.builder(DateTimeUnit.DAY_OF_MONTH).build(), 1, 7);
-        roundingInfos[4] = new RoundingInfo(Rounding.builder(DateTimeUnit.MONTH_OF_YEAR).build(), 1, 3);
-        roundingInfos[5] = new RoundingInfo(Rounding.builder(DateTimeUnit.YEAR_OF_CENTURY).build(), 1, 10, 20, 50, 100);
     }
 
     @Override
@@ -66,6 +56,8 @@ protected InternalAutoDateHistogram createTestInstance(String name,
                                                        List<PipelineAggregator> pipelineAggregators,
                                                        Map<String, Object> metaData,
                                                        InternalAggregations aggregations) {
+
+        roundingInfos = AutoDateHistogramAggregationBuilder.buildRoundings(null);
         int nbBuckets = randomNumberOfBuckets();
         int targetBuckets = randomIntBetween(1, nbBuckets * 2 + 1);
         List<InternalAutoDateHistogram.Bucket> buckets = new ArrayList<>(nbBuckets);
@@ -81,6 +73,7 @@ protected InternalAutoDateHistogram createTestInstance(String name,
         InternalAggregations subAggregations = new InternalAggregations(Collections.emptyList());
         BucketInfo bucketInfo = new BucketInfo(roundingInfos, randomIntBetween(0, roundingInfos.length - 1), subAggregations);
 
+
         return new InternalAutoDateHistogram(name, buckets, targetBuckets, bucketInfo, format, pipelineAggregators, metaData);
     }
 
@@ -92,13 +85,50 @@ protected void assertReduced(InternalAutoDateHistogram reduced, List<InternalAut
                 roundingIdx = histogram.getBucketInfo().roundingIdx;
             }
         }
-        Map<Long, Long> expectedCounts = new TreeMap<>();
-        for (Histogram histogram : inputs) {
+        RoundingInfo roundingInfo = roundingInfos[roundingIdx];
+
+        long lowest = Long.MAX_VALUE;
+        long highest = 0;
+        for (InternalAutoDateHistogram histogram : inputs) {
             for (Histogram.Bucket bucket : histogram.getBuckets()) {
-                expectedCounts.compute(roundingInfos[roundingIdx].rounding.round(((DateTime) bucket.getKey()).getMillis()),
-                        (key, oldValue) -> (oldValue == null ? 0 : oldValue) + bucket.getDocCount());
+                long bucketKey = ((DateTime) bucket.getKey()).getMillis();
+                if (bucketKey < lowest) {
+                    lowest = bucketKey;
+                }
+                if (bucketKey > highest) {
+                    highest = bucketKey;
+                }
             }
         }
+        long normalizedDuration = (highest - lowest) / roundingInfo.getRoughEstimateDurationMillis();
+        long innerIntervalToUse = 0;
+        for (int interval : roundingInfo.innerIntervals) {
+            if (normalizedDuration / interval < maxNumberOfBuckets()) {
+                innerIntervalToUse = interval;
+            }
+        }
+        Map<Long, Long> expectedCounts = new TreeMap<>();
+        long intervalInMillis = innerIntervalToUse*roundingInfo.getRoughEstimateDurationMillis();
+        for (long keyForBucket = roundingInfo.rounding.round(lowest);
+             keyForBucket <= highest;
+             keyForBucket = keyForBucket + intervalInMillis) {
+            expectedCounts.put(keyForBucket, 0L);
+
+            for (InternalAutoDateHistogram histogram : inputs) {
+                for (Histogram.Bucket bucket : histogram.getBuckets()) {
+                    long bucketKey = ((DateTime) bucket.getKey()).getMillis();
+                    long roundedBucketKey = roundingInfo.rounding.round(bucketKey);
+                    if (roundedBucketKey >= keyForBucket
+                        && roundedBucketKey < keyForBucket + intervalInMillis) {
+                        long count = bucket.getDocCount();
+                        expectedCounts.compute(keyForBucket,
+                            (key, oldValue) -> (oldValue == null ? 0 : oldValue) + count);
+                    }
+                }
+            }
+        }
+
+
         Map<Long, Long> actualCounts = new TreeMap<>();
         for (Histogram.Bucket bucket : reduced.getBuckets()) {
             actualCounts.compute(((DateTime) bucket.getKey()).getMillis(),
@@ -107,6 +137,12 @@ protected void assertReduced(InternalAutoDateHistogram reduced, List<InternalAut
         assertEquals(expectedCounts, actualCounts);
     }
 
+    @Override
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32215")
+    public void testReduceRandom() {
+        super.testReduceRandom();
+    }
+    
     @Override
     protected Writeable.Reader<InternalAutoDateHistogram> instanceReader() {
         return InternalAutoDateHistogram::new;
@@ -117,12 +153,6 @@ protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
         return ParsedAutoDateHistogram.class;
     }
 
-    @Override
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32215")
-    public void testReduceRandom() {
-        super.testReduceRandom();
-    }
-
     @Override
     protected InternalAutoDateHistogram mutateInstance(InternalAutoDateHistogram instance) {
         String name = instance.getName();
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsTests.java
index 985d82d4e1bee..755cb6e85292d 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsTests.java
@@ -57,7 +57,10 @@ protected InternalSignificantTerms createTestInstance(String name,
         Set<Long> terms = new HashSet<>();
         for (int i = 0; i < numBuckets; ++i) {
             long term = randomValueOtherThanMany(l -> terms.add(l) == false, random()::nextLong);
-            buckets.add(new SignificantLongTerms.Bucket(subsetDfs[i], subsetSize, supersetDfs[i], supersetSize, term, aggs, format));
+            SignificantLongTerms.Bucket bucket = new SignificantLongTerms.Bucket(subsetDfs[i], subsetSize, 
+                    supersetDfs[i], supersetSize, term, aggs, format);
+            bucket.updateScore(significanceHeuristic);
+            buckets.add(bucket);
         }
         return new SignificantLongTerms(name, requiredSize, 1L, pipelineAggregators, metaData, format, subsetSize,
                 supersetSize, significanceHeuristic, buckets);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsTests.java
index 5dafc1e846111..2255373fd346d 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsTests.java
@@ -50,7 +50,10 @@ protected InternalSignificantTerms createTestInstance(String name,
         Set<BytesRef> terms = new HashSet<>();
         for (int i = 0; i < numBuckets; ++i) {
             BytesRef term = randomValueOtherThanMany(b -> terms.add(b) == false, () -> new BytesRef(randomAlphaOfLength(10)));
-            buckets.add(new SignificantStringTerms.Bucket(term, subsetDfs[i], subsetSize, supersetDfs[i], supersetSize, aggs, format));
+            SignificantStringTerms.Bucket bucket = new SignificantStringTerms.Bucket(term, subsetDfs[i], subsetSize, 
+                    supersetDfs[i], supersetSize, aggs, format);
+            bucket.updateScore(significanceHeuristic);
+            buckets.add(bucket);
         }
         return new SignificantStringTerms(name, requiredSize, 1L, pipelineAggregators, metaData, format, subsetSize,
                 supersetSize, significanceHeuristic, buckets);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
index d74e43327fd03..1b33ed478709b 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
@@ -50,9 +50,11 @@
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.script.Script;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.BucketOrder;
@@ -67,6 +69,7 @@
 import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits;
 import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
@@ -83,6 +86,8 @@
 import java.util.function.Function;
 
 import static org.elasticsearch.index.mapper.SeqNoFieldMapper.PRIMARY_TERM_NAME;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.bucketScript;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
@@ -1060,6 +1065,34 @@ public void testWithNestedAggregations() throws IOException {
         }
     }
 
+    public void testOrderByPipelineAggregation() throws Exception {
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
+                    IndexSearcher indexSearcher = newIndexSearcher(indexReader);
+
+                    BucketScriptPipelineAggregationBuilder bucketScriptAgg = bucketScript(
+                        "script", new Script("2.718"));
+                    TermsAggregationBuilder termsAgg = terms("terms")
+                        .field("field")
+                        .valueType(ValueType.STRING)
+                        .order(BucketOrder.aggregation("script", true))
+                        .subAggregation(bucketScriptAgg);
+
+                    MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
+                    fieldType.setName("field");
+                    fieldType.setHasDocValues(true);
+
+                    AggregationExecutionException e = expectThrows(AggregationExecutionException.class,
+                        () -> createAggregator(termsAgg, indexSearcher, fieldType));
+                    assertEquals("Invalid aggregator order path [script]. The provided aggregation [script] " +
+                        "either does not exist, or is a pipeline aggregation and cannot be used to sort the buckets.",
+                        e.getMessage());
+                }
+            }
+        }
+    }
+
     private final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
     private List<Document> generateDocsWithNested(String id, int value, int[] nestedValues) {
         List<Document> documents = new ArrayList<>();
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
index 398044edcaf53..9e85455d96de9 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
@@ -117,6 +117,8 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
                 return value0 + value1 + value2;
             });
 
+            scripts.put("return null", vars -> null);
+
             return scripts;
         }
     }
@@ -478,6 +480,33 @@ public void testInlineScriptInsertZeros() {
         }
     }
 
+    public void testInlineScriptReturnNull() {
+        SearchResponse response = client()
+            .prepareSearch("idx")
+            .addAggregation(
+                histogram("histo")
+                    .field(FIELD_1_NAME).interval(interval)
+                    .subAggregation(
+                        bucketScript(
+                            "nullField",
+                            new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "return null", Collections.emptyMap())
+                        )
+                    )
+            ).execute().actionGet();
+
+        assertSearchResponse(response);
+
+        Histogram histo = response.getAggregations().get("histo");
+        assertThat(histo, notNullValue());
+        assertThat(histo.getName(), equalTo("histo"));
+        List<? extends Histogram.Bucket> buckets = histo.getBuckets();
+
+        for (int i = 0; i < buckets.size(); ++i) {
+            Histogram.Bucket bucket = buckets.get(i);
+            assertNull(bucket.getAggregations().get("nullField"));
+        }
+    }
+
     public void testStoredScript() {
         assertAcked(client().admin().cluster().preparePutStoredScript()
                 .setId("my_script")
diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
index 31fa4f838dfff..aea0243a399d4 100644
--- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
@@ -47,15 +47,10 @@
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.ReadableDateTime;
-import org.joda.time.base.BaseDateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
 
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
@@ -64,6 +59,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
@@ -115,7 +111,7 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             scripts.put("doc['date'].date.millis", vars -> {
                 Map<?, ?> doc = (Map) vars.get("doc");
                 ScriptDocValues.Dates dates = (ScriptDocValues.Dates) doc.get("date");
-                return dates.getValue().getMillis();
+                return ((ZonedDateTime) dates.getValue()).toInstant().toEpochMilli();
             });
 
             scripts.put("_fields['num1'].value", vars -> fieldsScript(vars, "num1"));
@@ -805,8 +801,8 @@ public void testDocValueFields() throws Exception {
         assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L));
         assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0));
         assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d));
-        BaseDateTime dateField = searchResponse.getHits().getAt(0).getFields().get("date_field").getValue();
-        assertThat(dateField.getMillis(), equalTo(date.toInstant().toEpochMilli()));
+        ZonedDateTime dateField = searchResponse.getHits().getAt(0).getFields().get("date_field").getValue();
+        assertThat(dateField.toInstant().toEpochMilli(), equalTo(date.toInstant().toEpochMilli()));
         assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
         assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
         assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
@@ -946,10 +942,10 @@ public void testDocValueFieldsWithFieldAlias() throws Exception {
         assertAcked(prepareCreate("test").addMapping("type", mapping));
         ensureGreen("test");
 
-        DateTime date = new DateTime(1990, 12, 29, 0, 0, DateTimeZone.UTC);
-        DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd");
+        ZonedDateTime date = ZonedDateTime.of(1990, 12, 29, 0, 0, 0, 0, ZoneOffset.UTC);
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.ROOT);
 
-        index("test", "type", "1", "text_field", "foo", "date_field", formatter.print(date));
+        index("test", "type", "1", "text_field", "foo", "date_field", formatter.format(date));
         refresh("test");
 
         SearchRequestBuilder builder = client().prepareSearch().setQuery(matchAllQuery())
@@ -977,7 +973,7 @@ public void testDocValueFieldsWithFieldAlias() throws Exception {
         DocumentField dateField = fields.get("date_field");
         assertThat(dateField.getName(), equalTo("date_field"));
 
-        ReadableDateTime fetchedDate = dateField.getValue();
+        ZonedDateTime fetchedDate = dateField.getValue();
         assertThat(fetchedDate, equalTo(date));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
index 6de15146faee5..da7f5f8a4cf2c 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
@@ -35,8 +35,10 @@
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.search.profile.query.RandomQueryGenerator.randomQueryBuilder;
 import static org.hamcrest.Matchers.equalTo;
@@ -149,6 +151,10 @@ public void testProfileMatchesRegular() throws Exception {
             SearchResponse vanillaResponse = responses[0].getResponse();
             SearchResponse profileResponse = responses[1].getResponse();
 
+            assertThat(vanillaResponse.getFailedShards(), equalTo(0));
+            assertThat(profileResponse.getFailedShards(), equalTo(0));
+            assertThat(vanillaResponse.getSuccessfulShards(), equalTo(profileResponse.getSuccessfulShards()));
+
             float vanillaMaxScore = vanillaResponse.getHits().getMaxScore();
             float profileMaxScore = profileResponse.getHits().getMaxScore();
             if (Float.isNaN(vanillaMaxScore)) {
@@ -159,10 +165,19 @@ public void testProfileMatchesRegular() throws Exception {
                         vanillaMaxScore, profileMaxScore, 0.001);
             }
 
-            assertThat(
-                    "Profile totalHits of [" + profileResponse.getHits().getTotalHits() + "] is not close to Vanilla totalHits ["
-                            + vanillaResponse.getHits().getTotalHits() + "]",
-                    vanillaResponse.getHits().getTotalHits(), equalTo(profileResponse.getHits().getTotalHits()));
+            if (vanillaResponse.getHits().totalHits != profileResponse.getHits().totalHits) {
+                Set<SearchHit> vanillaSet = new HashSet<>(Arrays.asList(vanillaResponse.getHits().getHits()));
+                Set<SearchHit> profileSet = new HashSet<>(Arrays.asList(profileResponse.getHits().getHits()));
+                if (vanillaResponse.getHits().totalHits > profileResponse.getHits().totalHits) {
+                    vanillaSet.removeAll(profileSet);
+                    fail("Vanilla hits were larger than profile hits.  Non-overlapping elements were: "
+                        + vanillaSet.toString());
+                } else {
+                    profileSet.removeAll(vanillaSet);
+                    fail("Profile hits were larger than vanilla hits.  Non-overlapping elements were: "
+                        + profileSet.toString());
+                }
+            }
 
             SearchHit[] vanillaHits = vanillaResponse.getHits().getHits();
             SearchHit[] profileHits = profileResponse.getHits().getHits();
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java b/server/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
deleted file mode 100644
index 63c8aa1e66816..0000000000000
--- a/server/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.suggest;
-
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.query.QueryShardContext;
-
-import java.io.IOException;
-import java.util.Locale;
-import java.util.Map;
-
-public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestionsContext> {
-
-    public static final CustomSuggester INSTANCE = new CustomSuggester();
-
-    // This is a pretty dumb implementation which returns the original text + fieldName + custom config option + 12 or 123
-    @Override
-    public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> innerExecute(String name, CustomSuggestionsContext suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
-        // Get the suggestion context
-        String text = suggestion.getText().utf8ToString();
-
-        // create two suggestions with 12 and 123 appended
-        Suggest.Suggestion<Suggest.Suggestion.Entry<Suggest.Suggestion.Entry.Option>> response = new Suggest.Suggestion<>(name, suggestion.getSize());
-
-        String firstSuggestion = String.format(Locale.ROOT, "%s-%s-%s-%s", text, suggestion.getField(), suggestion.options.get("suffix"), "12");
-        Suggest.Suggestion.Entry<Suggest.Suggestion.Entry.Option> resultEntry12 = new Suggest.Suggestion.Entry<>(new Text(firstSuggestion), 0, text.length() + 2);
-        response.addTerm(resultEntry12);
-
-        String secondSuggestion = String.format(Locale.ROOT, "%s-%s-%s-%s", text, suggestion.getField(), suggestion.options.get("suffix"), "123");
-        Suggest.Suggestion.Entry<Suggest.Suggestion.Entry.Option> resultEntry123 = new Suggest.Suggestion.Entry<>(new Text(secondSuggestion), 0, text.length() + 3);
-        response.addTerm(resultEntry123);
-
-        return response;
-    }
-
-    public static class CustomSuggestionsContext extends SuggestionSearchContext.SuggestionContext {
-
-        public Map<String, Object> options;
-
-        public CustomSuggestionsContext(QueryShardContext context, Map<String, Object> options) {
-            super(new CustomSuggester(), context);
-            this.options = options;
-        }
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java b/server/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
deleted file mode 100644
index a6040a0c65f84..0000000000000
--- a/server/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.suggest;
-
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.BytesRefs;
-import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.SearchPlugin;
-import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-
-import static java.util.Collections.singletonList;
-import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
-
-/**
- * Integration test for registering a custom suggester.
- */
-@ClusterScope(scope= Scope.SUITE, numDataNodes =1)
-public class CustomSuggesterSearchIT extends ESIntegTestCase {
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(CustomSuggesterPlugin.class);
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return Arrays.asList(CustomSuggesterPlugin.class);
-    }
-
-    public static class CustomSuggesterPlugin extends Plugin implements SearchPlugin {
-        @Override
-        public List<SuggesterSpec<?>> getSuggesters() {
-            return singletonList(new SuggesterSpec<CustomSuggestionBuilder>("custom", CustomSuggestionBuilder::new,
-                    CustomSuggestionBuilder::fromXContent));
-        }
-    }
-
-    public void testThatCustomSuggestersCanBeRegisteredAndWork() throws Exception {
-        createIndex("test");
-        client().prepareIndex("test", "test", "1").setSource(jsonBuilder()
-                .startObject()
-                .field("name", "arbitrary content")
-                .endObject())
-                .setRefreshPolicy(IMMEDIATE).get();
-
-        String randomText = randomAlphaOfLength(10);
-        String randomField = randomAlphaOfLength(10);
-        String randomSuffix = randomAlphaOfLength(10);
-        SuggestBuilder suggestBuilder = new SuggestBuilder();
-        suggestBuilder.addSuggestion("someName", new CustomSuggestionBuilder(randomField, randomSuffix).text(randomText));
-        SearchRequestBuilder searchRequestBuilder = client().prepareSearch("test").setTypes("test").setFrom(0).setSize(1)
-                .suggest(suggestBuilder);
-
-        SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
-
-        // TODO: infer type once JI-9019884 is fixed
-        // TODO: see also JDK-8039214
-        List<Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> suggestions =
-            CollectionUtils.<Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>iterableAsArrayList(
-                searchResponse.getSuggest().getSuggestion("someName"));
-        assertThat(suggestions, hasSize(2));
-        assertThat(suggestions.get(0).getText().string(),
-            is(String.format(Locale.ROOT, "%s-%s-%s-12", randomText, randomField, randomSuffix)));
-        assertThat(suggestions.get(1).getText().string(),
-            is(String.format(Locale.ROOT, "%s-%s-%s-123", randomText, randomField, randomSuffix)));
-    }
-
-    public static class CustomSuggestionBuilder extends SuggestionBuilder<CustomSuggestionBuilder> {
-        protected static final ParseField RANDOM_SUFFIX_FIELD = new ParseField("suffix");
-
-        private String randomSuffix;
-
-        public CustomSuggestionBuilder(String randomField, String randomSuffix) {
-            super(randomField);
-            this.randomSuffix = randomSuffix;
-        }
-
-        /**
-         * Read from a stream.
-         */
-        public CustomSuggestionBuilder(StreamInput in) throws IOException {
-            super(in);
-            this.randomSuffix = in.readString();
-        }
-
-        @Override
-        public void doWriteTo(StreamOutput out) throws IOException {
-            out.writeString(randomSuffix);
-        }
-
-        @Override
-        protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(RANDOM_SUFFIX_FIELD.getPreferredName(), randomSuffix);
-            return builder;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return "custom";
-        }
-
-        @Override
-        protected boolean doEquals(CustomSuggestionBuilder other) {
-            return Objects.equals(randomSuffix, other.randomSuffix);
-        }
-
-        @Override
-        protected int doHashCode() {
-            return Objects.hash(randomSuffix);
-        }
-
-        public static CustomSuggestionBuilder fromXContent(XContentParser parser) throws IOException {
-            XContentParser.Token token;
-            String currentFieldName = null;
-            String fieldname = null;
-            String suffix = null;
-            String analyzer = null;
-            int sizeField = -1;
-            int shardSize = -1;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                } else if (token.isValue()) {
-                    if (SuggestionBuilder.ANALYZER_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                        analyzer = parser.text();
-                    } else if (SuggestionBuilder.FIELDNAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                        fieldname = parser.text();
-                    } else if (SuggestionBuilder.SIZE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                        sizeField = parser.intValue();
-                    } else if (SuggestionBuilder.SHARDSIZE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                        shardSize = parser.intValue();
-                    } else if (RANDOM_SUFFIX_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                        suffix = parser.text();
-                    }
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(),
-                                               "suggester[custom] doesn't support field [" + currentFieldName + "]");
-                }
-            }
-
-            // now we should have field name, check and copy fields over to the suggestion builder we return
-            if (fieldname == null) {
-                throw new ParsingException(parser.getTokenLocation(), "the required field option is missing");
-            }
-            CustomSuggestionBuilder builder = new CustomSuggestionBuilder(fieldname, suffix);
-            if (analyzer != null) {
-                builder.analyzer(analyzer);
-            }
-            if (sizeField != -1) {
-                builder.size(sizeField);
-            }
-            if (shardSize != -1) {
-                builder.shardSize(shardSize);
-            }
-            return builder;
-        }
-
-        @Override
-        public SuggestionContext build(QueryShardContext context) throws IOException {
-            Map<String, Object> options = new HashMap<>();
-            options.put(FIELDNAME_FIELD.getPreferredName(), field());
-            options.put(RANDOM_SUFFIX_FIELD.getPreferredName(), randomSuffix);
-            CustomSuggester.CustomSuggestionsContext customSuggestionsContext =
-                new CustomSuggester.CustomSuggestionsContext(context, options);
-            customSuggestionsContext.setField(field());
-            assert text != null;
-            customSuggestionsContext.setText(BytesRefs.toBytesRef(text));
-            return customSuggestionsContext;
-        }
-
-    }
-
-}
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java b/server/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
index 412dd3c6a2017..903d06a372e72 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
@@ -19,9 +19,14 @@
 
 package org.elasticsearch.search.suggest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -30,6 +35,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
@@ -37,6 +43,7 @@
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
 import org.elasticsearch.search.suggest.term.TermSuggestion;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -44,6 +51,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import static java.util.Collections.emptyList;
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName;
@@ -114,10 +122,11 @@ public void testFromXContent() throws IOException {
     }
 
     public void testToXContent() throws IOException {
-        Option option = new Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
-        Entry<Option> entry = new Entry<>(new Text("entryText"), 42, 313);
+        PhraseSuggestion.Entry.Option option = new PhraseSuggestion.Entry.Option(new Text("someText"), new Text("somethingHighlighted"),
+            1.3f, true);
+        PhraseSuggestion.Entry entry = new PhraseSuggestion.Entry(new Text("entryText"), 42, 313);
         entry.addOption(option);
-        Suggestion<Entry<Option>> suggestion = new Suggestion<>("suggestionName", 5);
+        PhraseSuggestion suggestion = new PhraseSuggestion("suggestionName", 5);
         suggestion.addTerm(entry);
         Suggest suggest = new Suggest(Collections.singletonList(suggestion));
         BytesReference xContent = toXContent(suggest, XContentType.JSON, randomBoolean());
@@ -196,9 +205,9 @@ public void testMergingSuggestionOptions() {
         String secondWord = randomAlphaOfLength(10);
         Text suggestionText = new Text(suggestedWord + " " + secondWord);
         Text highlighted = new Text("<em>" + suggestedWord + "</em> " + secondWord);
-        PhraseSuggestion.Entry.Option option1 = new Option(suggestionText, highlighted, 0.7f, false);
-        PhraseSuggestion.Entry.Option option2 = new Option(suggestionText, highlighted, 0.8f, true);
-        PhraseSuggestion.Entry.Option option3 = new Option(suggestionText, highlighted, 0.6f);
+        PhraseSuggestion.Entry.Option option1 = new PhraseSuggestion.Entry.Option(suggestionText, highlighted, 0.7f, false);
+        PhraseSuggestion.Entry.Option option2 = new PhraseSuggestion.Entry.Option(suggestionText, highlighted, 0.8f, true);
+        PhraseSuggestion.Entry.Option option3 = new PhraseSuggestion.Entry.Option(suggestionText, highlighted, 0.6f);
         assertEquals(suggestionText, option1.getText());
         assertEquals(highlighted, option1.getHighlighted());
         assertFalse(option1.collateMatch());
@@ -214,4 +223,39 @@ public void testMergingSuggestionOptions() {
         assertTrue(option1.getScore() > 0.7f);
         assertTrue(option1.collateMatch());
     }
+
+    public void testSerialization() throws IOException {
+        final Version bwcVersion = VersionUtils.randomVersionBetween(random(),
+            Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT);
+
+        final Suggest suggest = createTestItem();
+        final Suggest bwcSuggest;
+
+        NamedWriteableRegistry registry = new NamedWriteableRegistry
+            (new SearchModule(Settings.EMPTY, false, emptyList()).getNamedWriteables());
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(bwcVersion);
+            suggest.writeTo(out);
+            try (NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry)) {
+                in.setVersion(bwcVersion);
+                bwcSuggest = new Suggest(in);
+            }
+        }
+
+        assertEquals(suggest, bwcSuggest);
+
+        final Suggest backAgain;
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(Version.CURRENT);
+            bwcSuggest.writeTo(out);
+            try (NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry)) {
+                in.setVersion(Version.CURRENT);
+                backAgain = new Suggest(in);
+            }
+        }
+
+        assertEquals(suggest, backAgain);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/SuggestionEntryTests.java b/server/src/test/java/org/elasticsearch/search/suggest/SuggestionEntryTests.java
index 46971a5537b26..ac2aff2957fdd 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/SuggestionEntryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/SuggestionEntryTests.java
@@ -129,10 +129,12 @@ private void doTestFromXContent(boolean addRandomFields) throws IOException {
     }
 
     public void testToXContent() throws IOException {
-        Option option = new Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
-        Entry<Option> entry = new Entry<>(new Text("entryText"), 42, 313);
-        entry.addOption(option);
-        BytesReference xContent = toXContent(entry, XContentType.JSON, randomBoolean());
+        PhraseSuggestion.Entry.Option phraseOption = new PhraseSuggestion.Entry.Option(new Text("someText"),
+                new Text("somethingHighlighted"),
+            1.3f, true);
+        PhraseSuggestion.Entry phraseEntry = new PhraseSuggestion.Entry(new Text("entryText"), 42, 313);
+        phraseEntry.addOption(phraseOption);
+        BytesReference xContent = toXContent(phraseEntry, XContentType.JSON, randomBoolean());
         assertEquals(
                 "{\"text\":\"entryText\","
                 + "\"offset\":42,"
@@ -144,11 +146,10 @@ public void testToXContent() throws IOException {
                     + "\"collate_match\":true}"
                 + "]}", xContent.utf8ToString());
 
-        org.elasticsearch.search.suggest.term.TermSuggestion.Entry.Option termOption =
-                new org.elasticsearch.search.suggest.term.TermSuggestion.Entry.Option(new Text("termSuggestOption"), 42, 3.13f);
-        entry = new Entry<>(new Text("entryText"), 42, 313);
-        entry.addOption(termOption);
-        xContent = toXContent(entry, XContentType.JSON, randomBoolean());
+        TermSuggestion.Entry.Option termOption = new TermSuggestion.Entry.Option(new Text("termSuggestOption"), 42, 3.13f);
+        TermSuggestion.Entry termEntry = new TermSuggestion.Entry(new Text("entryText"), 42, 313);
+        termEntry.addOption(termOption);
+        xContent = toXContent(termEntry, XContentType.JSON, randomBoolean());
         assertEquals(
                 "{\"text\":\"entryText\","
                 + "\"offset\":42,"
@@ -159,12 +160,11 @@ public void testToXContent() throws IOException {
                     + "\"freq\":42}"
                 + "]}", xContent.utf8ToString());
 
-        org.elasticsearch.search.suggest.completion.CompletionSuggestion.Entry.Option completionOption =
-                new org.elasticsearch.search.suggest.completion.CompletionSuggestion.Entry.Option(-1, new Text("completionOption"),
+        CompletionSuggestion.Entry.Option completionOption = new CompletionSuggestion.Entry.Option(-1, new Text("completionOption"),
                         3.13f, Collections.singletonMap("key", Collections.singleton("value")));
-        entry = new Entry<>(new Text("entryText"), 42, 313);
-        entry.addOption(completionOption);
-        xContent = toXContent(entry, XContentType.JSON, randomBoolean());
+        CompletionSuggestion.Entry completionEntry = new CompletionSuggestion.Entry(new Text("entryText"), 42, 313);
+        completionEntry.addOption(completionOption);
+        xContent = toXContent(completionEntry, XContentType.JSON, randomBoolean());
         assertEquals(
                 "{\"text\":\"entryText\","
                 + "\"offset\":42,"
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java b/server/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java
index 7385dd38f45b4..37d886fd5d825 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -41,7 +42,7 @@ public static Option createTestItem() {
         float score = randomFloat();
         Text highlighted = randomFrom((Text) null, new Text(randomAlphaOfLengthBetween(5, 15)));
         Boolean collateMatch = randomFrom((Boolean) null, randomBoolean());
-        return new Option(text, highlighted, score, collateMatch);
+        return new PhraseSuggestion.Entry.Option(text, highlighted, score, collateMatch);
     }
 
     public void testFromXContent() throws IOException {
@@ -66,7 +67,7 @@ private void doTestFromXContent(boolean addRandomFields) throws IOException {
         Option parsed;
         try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
             ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
-            parsed = Option.fromXContent(parser);
+            parsed = PhraseSuggestion.Entry.Option.fromXContent(parser);
             assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
             assertNull(parser.nextToken());
         }
@@ -78,7 +79,7 @@ private void doTestFromXContent(boolean addRandomFields) throws IOException {
     }
 
     public void testToXContent() throws IOException {
-        Option option = new Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
+        Option option = new PhraseSuggestion.Entry.Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
         BytesReference xContent = toXContent(option, XContentType.JSON, randomBoolean());
         assertEquals("{\"text\":\"someText\","
                       + "\"highlighted\":\"somethingHighlighted\","
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java b/server/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java
index c8384a948a66c..55c8048baf6bd 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java
@@ -188,14 +188,15 @@ public void testUnknownSuggestionTypeThrows() throws IOException {
     public void testToXContent() throws IOException {
         ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
         {
-            Option option = new Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
-            Entry<Option> entry = new Entry<>(new Text("entryText"), 42, 313);
+            PhraseSuggestion.Entry.Option option = new PhraseSuggestion.Entry.Option(new Text("someText"), new Text("somethingHighlighted"),
+                1.3f, true);
+            PhraseSuggestion.Entry entry = new PhraseSuggestion.Entry(new Text("entryText"), 42, 313);
             entry.addOption(option);
-            Suggestion<Entry<Option>> suggestion = new Suggestion<>("suggestionName", 5);
+            PhraseSuggestion suggestion = new PhraseSuggestion("suggestionName", 5);
             suggestion.addTerm(entry);
             BytesReference xContent = toXContent(suggestion, XContentType.JSON, params, randomBoolean());
             assertEquals(
-                    "{\"suggestion#suggestionName\":[{"
+                    "{\"phrase#suggestionName\":[{"
                             + "\"text\":\"entryText\","
                             + "\"offset\":42,"
                             + "\"length\":313,"
@@ -208,7 +209,8 @@ public void testToXContent() throws IOException {
                     + "}", xContent.utf8ToString());
         }
         {
-            Option option = new Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
+            PhraseSuggestion.Entry.Option option = new PhraseSuggestion.Entry.Option(new Text("someText"), new Text("somethingHighlighted"),
+                    1.3f, true);
             PhraseSuggestion.Entry entry = new PhraseSuggestion.Entry(new Text("entryText"), 42, 313, 1.0);
             entry.addOption(option);
             PhraseSuggestion suggestion = new PhraseSuggestion("suggestionName", 5);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index 5d2abdd149223..a008846e2d325 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.AdminClient;
@@ -40,8 +39,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.NamedDiff;
-import org.elasticsearch.cluster.RestoreInProgress;
-import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -165,24 +162,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(MockRepository.Plugin.class, TestCustomMetaDataPlugin.class);
     }
 
-    public void testClusterStateHasCustoms() throws Exception {
-        ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().all().get();
-        assertNotNull(clusterStateResponse.getState().custom(SnapshotsInProgress.TYPE));
-        assertNotNull(clusterStateResponse.getState().custom(RestoreInProgress.TYPE));
-        assertNotNull(clusterStateResponse.getState().custom(SnapshotDeletionsInProgress.TYPE));
-        internalCluster().ensureAtLeastNumDataNodes(2);
-        if (randomBoolean()) {
-            internalCluster().fullRestart();
-        } else {
-            internalCluster().rollingRestart();
-        }
-
-        clusterStateResponse = client().admin().cluster().prepareState().all().get();
-        assertNotNull(clusterStateResponse.getState().custom(SnapshotsInProgress.TYPE));
-        assertNotNull(clusterStateResponse.getState().custom(RestoreInProgress.TYPE));
-        assertNotNull(clusterStateResponse.getState().custom(SnapshotDeletionsInProgress.TYPE));
-    }
-
     public void testRestorePersistentSettings() throws Exception {
         logger.info("--> start 2 nodes");
         internalCluster().startNode();
diff --git a/server/src/test/java/org/elasticsearch/transport/ConnectionManagerTests.java b/server/src/test/java/org/elasticsearch/transport/ConnectionManagerTests.java
new file mode 100644
index 0000000000000..64e8a42600458
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/transport/ConnectionManagerTests.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ConnectionManagerTests extends ESTestCase {
+
+    private ConnectionManager connectionManager;
+    private ThreadPool threadPool;
+    private Transport transport;
+    private ConnectionProfile connectionProfile;
+
+    @Before
+    public void createConnectionManager() {
+        Settings settings = Settings.builder()
+            .put("node.name", ConnectionManagerTests.class.getSimpleName())
+            .build();
+        threadPool = new ThreadPool(settings);
+        transport = mock(Transport.class);
+        connectionManager = new ConnectionManager(settings, transport, threadPool);
+        TimeValue oneSecond = new TimeValue(1000);
+        connectionProfile = ConnectionProfile.buildSingleChannelProfile(TransportRequestOptions.Type.REG, oneSecond, oneSecond);
+    }
+
+    @After
+    public void stopThreadPool() {
+        threadPool.shutdown();
+    }
+
+    public void testConnectAndDisconnect() {
+        AtomicInteger nodeConnectedCount = new AtomicInteger();
+        AtomicInteger nodeDisconnectedCount = new AtomicInteger();
+        connectionManager.addListener(new TransportConnectionListener() {
+            @Override
+            public void onNodeConnected(DiscoveryNode node) {
+                nodeConnectedCount.incrementAndGet();
+            }
+
+            @Override
+            public void onNodeDisconnected(DiscoveryNode node) {
+                nodeDisconnectedCount.incrementAndGet();
+            }
+        });
+
+
+        DiscoveryNode node = new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 0), Version.CURRENT);
+        Transport.Connection connection = new TestConnect(node);
+        when(transport.openConnection(node, connectionProfile)).thenReturn(connection);
+
+        assertFalse(connectionManager.nodeConnected(node));
+
+        AtomicReference<Transport.Connection> connectionRef = new AtomicReference<>();
+        CheckedBiConsumer<Transport.Connection, ConnectionProfile, IOException> validator = (c, p) -> connectionRef.set(c);
+        connectionManager.connectToNode(node, connectionProfile, validator);
+
+        assertFalse(connection.isClosed());
+        assertTrue(connectionManager.nodeConnected(node));
+        assertSame(connection, connectionManager.getConnection(node));
+        assertEquals(1, connectionManager.connectedNodeCount());
+        assertEquals(1, nodeConnectedCount.get());
+        assertEquals(0, nodeDisconnectedCount.get());
+
+        if (randomBoolean()) {
+            connectionManager.disconnectFromNode(node);
+        } else {
+            connection.close();
+        }
+        assertTrue(connection.isClosed());
+        assertEquals(0, connectionManager.connectedNodeCount());
+        assertEquals(1, nodeConnectedCount.get());
+        assertEquals(1, nodeDisconnectedCount.get());
+    }
+
+    public void testConnectFails() {
+        AtomicInteger nodeConnectedCount = new AtomicInteger();
+        AtomicInteger nodeDisconnectedCount = new AtomicInteger();
+        connectionManager.addListener(new TransportConnectionListener() {
+            @Override
+            public void onNodeConnected(DiscoveryNode node) {
+                nodeConnectedCount.incrementAndGet();
+            }
+
+            @Override
+            public void onNodeDisconnected(DiscoveryNode node) {
+                nodeDisconnectedCount.incrementAndGet();
+            }
+        });
+
+
+        DiscoveryNode node = new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 0), Version.CURRENT);
+        Transport.Connection connection = new TestConnect(node);
+        when(transport.openConnection(node, connectionProfile)).thenReturn(connection);
+
+        assertFalse(connectionManager.nodeConnected(node));
+
+        CheckedBiConsumer<Transport.Connection, ConnectionProfile, IOException> validator = (c, p) -> {
+            throw new ConnectTransportException(node, "");
+        };
+
+        expectThrows(ConnectTransportException.class, () -> connectionManager.connectToNode(node, connectionProfile, validator));
+
+        assertTrue(connection.isClosed());
+        assertFalse(connectionManager.nodeConnected(node));
+        expectThrows(NodeNotConnectedException.class, () -> connectionManager.getConnection(node));
+        assertEquals(0, connectionManager.connectedNodeCount());
+        assertEquals(0, nodeConnectedCount.get());
+        assertEquals(0, nodeDisconnectedCount.get());
+    }
+
+    private static class TestConnect extends CloseableConnection {
+
+        private final DiscoveryNode node;
+
+        private TestConnect(DiscoveryNode node) {
+            this.node = node;
+        }
+
+        @Override
+        public DiscoveryNode getNode() {
+            return node;
+        }
+
+        @Override
+        public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+            throws TransportException {
+
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
index 5522550f76c7a..8da4064d1c89a 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
@@ -105,34 +105,34 @@ public static MockTransportService startTransport(String id, List<DiscoveryNode>
     }
 
     public static MockTransportService startTransport(
-            final String id,
-            final List<DiscoveryNode> knownNodes,
-            final Version version,
-            final ThreadPool threadPool,
-            final Settings settings) {
+        final String id,
+        final List<DiscoveryNode> knownNodes,
+        final Version version,
+        final ThreadPool threadPool,
+        final Settings settings) {
         boolean success = false;
         final Settings s = Settings.builder().put(settings).put("node.name", id).build();
         ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(s);
         MockTransportService newService = MockTransportService.createNewService(s, version, threadPool, null);
         try {
-            newService.registerRequestHandler(ClusterSearchShardsAction.NAME,ThreadPool.Names.SAME, ClusterSearchShardsRequest::new,
+            newService.registerRequestHandler(ClusterSearchShardsAction.NAME, ThreadPool.Names.SAME, ClusterSearchShardsRequest::new,
                 (request, channel, task) -> {
-                        if ("index_not_found".equals(request.preference())) {
-                            channel.sendResponse(new IndexNotFoundException("index"));
-                        } else {
-                            channel.sendResponse(new ClusterSearchShardsResponse(new ClusterSearchShardsGroup[0],
-                                    knownNodes.toArray(new DiscoveryNode[0]), Collections.emptyMap()));
-                        }
-                    });
+                    if ("index_not_found".equals(request.preference())) {
+                        channel.sendResponse(new IndexNotFoundException("index"));
+                    } else {
+                        channel.sendResponse(new ClusterSearchShardsResponse(new ClusterSearchShardsGroup[0],
+                            knownNodes.toArray(new DiscoveryNode[0]), Collections.emptyMap()));
+                    }
+                });
             newService.registerRequestHandler(ClusterStateAction.NAME, ThreadPool.Names.SAME, ClusterStateRequest::new,
                 (request, channel, task) -> {
-                        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
-                        for (DiscoveryNode node : knownNodes) {
-                            builder.add(node);
-                        }
-                        ClusterState build = ClusterState.builder(clusterName).nodes(builder.build()).build();
-                        channel.sendResponse(new ClusterStateResponse(clusterName, build, 0L));
-                    });
+                    DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+                    for (DiscoveryNode node : knownNodes) {
+                        builder.add(node);
+                    }
+                    ClusterState build = ClusterState.builder(clusterName).nodes(builder.build()).build();
+                    channel.sendResponse(new ClusterStateResponse(clusterName, build, 0L));
+                });
             newService.start();
             newService.acceptIncomingRequests();
             success = true;
@@ -436,19 +436,28 @@ public void sendRequest(long requestId, String action, TransportRequest request,
                     }
 
                     @Override
-                    public void close() throws IOException {
+                    public void addCloseListener(ActionListener<Void> listener) {
                         // no-op
                     }
-                };
-                service.addDelegate(seedNode.getAddress(), new MockTransportService.DelegateTransport(service.getOriginalTransport()) {
+
                     @Override
-                    public Connection getConnection(DiscoveryNode node) {
-                        if (node == seedNode) {
-                            return seedConnection;
-                        }
-                        return super.getConnection(node);
+                    public boolean isClosed() {
+                        return false;
                     }
+
+                    @Override
+                    public void close() {
+                        // no-op
+                    }
+                };
+
+                service.addGetConnectionBehavior(seedNode.getAddress(), (connectionManager, discoveryNode) -> {
+                    if (discoveryNode == seedNode) {
+                        return seedConnection;
+                    }
+                    return connectionManager.getConnection(discoveryNode);
                 });
+
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
@@ -511,7 +520,9 @@ public void run() {
                 closeRemote.countDown();
                 listenerCalled.await();
                 assertNotNull(exceptionReference.get());
-                expectThrows(CancellableThreads.ExecutionCancelledException.class, () -> {throw exceptionReference.get();});
+                expectThrows(CancellableThreads.ExecutionCancelledException.class, () -> {
+                    throw exceptionReference.get();
+                });
 
             }
         }
@@ -545,7 +556,7 @@ public void testFetchShards() throws Exception {
                         .indicesOptions(request.indicesOptions()).local(true).preference(request.preference())
                         .routing(request.routing());
                     connection.fetchSearchShards(searchShardsRequest,
-                            new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
+                        new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
                     responseLatch.await();
                     assertNull(failReference.get());
                     assertNotNull(reference.get());
@@ -624,18 +635,18 @@ public void testFetchShardsSkipUnavailable() throws Exception {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                        Collections.singletonList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    Collections.singletonList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
 
                     SearchRequest request = new SearchRequest("test-index");
                     ClusterSearchShardsRequest searchShardsRequest = new ClusterSearchShardsRequest("test-index")
-                            .indicesOptions(request.indicesOptions()).local(true).preference(request.preference())
-                            .routing(request.routing());
+                        .indicesOptions(request.indicesOptions()).local(true).preference(request.preference())
+                        .routing(request.routing());
                     {
                         CountDownLatch responseLatch = new CountDownLatch(1);
                         AtomicReference<ClusterSearchShardsResponse> reference = new AtomicReference<>();
                         AtomicReference<Exception> failReference = new AtomicReference<>();
                         connection.fetchSearchShards(searchShardsRequest,
-                                new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
+                            new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
                         assertTrue(responseLatch.await(5, TimeUnit.SECONDS));
                         assertNull(failReference.get());
                         assertNotNull(reference.get());
@@ -664,7 +675,9 @@ public void onNodeDisconnected(DiscoveryNode node) {
                         AtomicReference<ClusterSearchShardsResponse> reference = new AtomicReference<>();
                         AtomicReference<Exception> failReference = new AtomicReference<>();
                         connection.fetchSearchShards(searchShardsRequest,
-                                new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
+                            new LatchedActionListener<>(ActionListener.wrap((s) -> {
+                                reference.set(s);
+                            }, failReference::set), responseLatch));
                         assertTrue(responseLatch.await(1, TimeUnit.SECONDS));
                         assertNotNull(failReference.get());
                         assertNull(reference.get());
@@ -677,7 +690,7 @@ public void onNodeDisconnected(DiscoveryNode node) {
                         AtomicReference<ClusterSearchShardsResponse> reference = new AtomicReference<>();
                         AtomicReference<Exception> failReference = new AtomicReference<>();
                         connection.fetchSearchShards(searchShardsRequest,
-                                new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
+                            new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
                         assertTrue(responseLatch.await(1, TimeUnit.SECONDS));
                         assertNull(failReference.get());
                         assertNotNull(reference.get());
@@ -700,7 +713,7 @@ public void onNodeDisconnected(DiscoveryNode node) {
                         AtomicReference<ClusterSearchShardsResponse> reference = new AtomicReference<>();
                         AtomicReference<Exception> failReference = new AtomicReference<>();
                         connection.fetchSearchShards(searchShardsRequest,
-                                new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
+                            new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
                         assertTrue(responseLatch.await(1, TimeUnit.SECONDS));
                         assertNull(failReference.get());
                         assertNotNull(reference.get());
@@ -747,27 +760,28 @@ public void run() {
                                     for (int i = 0; i < numConnectionAttempts; i++) {
                                         AtomicBoolean executed = new AtomicBoolean(false);
                                         ActionListener<Void> listener = ActionListener.wrap(
-                                                x -> {
-                                                    assertTrue(executed.compareAndSet(false, true));
-                                                    latch.countDown();},
-                                                x -> {
-                                                    /*
-                                                     * This can occur on a thread submitted to the thread pool while we are closing the
-                                                     * remote cluster connection at the end of the test.
-                                                     */
-                                                    if (x instanceof CancellableThreads.ExecutionCancelledException) {
-                                                        // we should already be shutting down
-                                                        assertTrue(executed.get());
-                                                        return;
-                                                    }
+                                            x -> {
+                                                assertTrue(executed.compareAndSet(false, true));
+                                                latch.countDown();
+                                            },
+                                            x -> {
+                                                /*
+                                                 * This can occur on a thread submitted to the thread pool while we are closing the
+                                                 * remote cluster connection at the end of the test.
+                                                 */
+                                                if (x instanceof CancellableThreads.ExecutionCancelledException) {
+                                                    // we should already be shutting down
+                                                    assertTrue(executed.get());
+                                                    return;
+                                                }
 
-                                                    assertTrue(executed.compareAndSet(false, true));
-                                                    latch.countDown();
+                                                assertTrue(executed.compareAndSet(false, true));
+                                                latch.countDown();
 
-                                                    if (!(x instanceof RejectedExecutionException)) {
-                                                        throw new AssertionError(x);
-                                                    }
-                                                });
+                                                if (!(x instanceof RejectedExecutionException)) {
+                                                    throw new AssertionError(x);
+                                                }
+                                            });
                                         connection.updateSeedNodes(seedNodes, listener);
                                     }
                                     latch.await();
@@ -854,7 +868,7 @@ public void run() {
                                                 } else {
                                                     throw new AssertionError(x);
                                                 }
-                                        });
+                                            });
                                         try {
                                             connection.updateSeedNodes(seedNodes, listener);
                                         } catch (Exception e) {
@@ -898,7 +912,7 @@ public void testGetConnectionInfo() throws Exception {
                 service.acceptIncomingRequests();
                 int maxNumConnections = randomIntBetween(1, 5);
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                        seedNodes, service, maxNumConnections, n -> true)) {
+                    seedNodes, service, maxNumConnections, n -> true)) {
                     // test no nodes connected
                     RemoteConnectionInfo remoteConnectionInfo = assertSerialization(connection.getConnectionInfo());
                     assertNotNull(remoteConnectionInfo);
@@ -973,16 +987,16 @@ private static RemoteConnectionInfo assertSerialization(RemoteConnectionInfo inf
             RemoteConnectionInfo remoteConnectionInfo = new RemoteConnectionInfo(in);
             assertEquals(info, remoteConnectionInfo);
             assertEquals(info.hashCode(), remoteConnectionInfo.hashCode());
-            return  randomBoolean() ? info : remoteConnectionInfo;
+            return randomBoolean() ? info : remoteConnectionInfo;
         }
     }
 
     public void testRemoteConnectionInfoBwComp() throws IOException {
         final Version version = VersionUtils.randomVersionBetween(random(),
-                Version.V_6_1_0, VersionUtils.getPreviousVersion(Version.V_7_0_0_alpha1));
+            Version.V_6_1_0, VersionUtils.getPreviousVersion(Version.V_7_0_0_alpha1));
         RemoteConnectionInfo expected = new RemoteConnectionInfo("test_cluster",
-                Collections.singletonList(new TransportAddress(TransportAddress.META_ADDRESS, 1)),
-                4, 4, new TimeValue(30, TimeUnit.MINUTES), false);
+            Collections.singletonList(new TransportAddress(TransportAddress.META_ADDRESS, 1)),
+            4, 4, new TimeValue(30, TimeUnit.MINUTES), false);
 
         // This version was created using the serialization code in use from 6.1 but before 7.0
         String encoded = "AQQAAAAABzAuMC4wLjAAAAABAQQAAAAABzAuMC4wLjAAAABQBDwEBAx0ZXN0X2NsdXN0ZXIA";
@@ -1007,7 +1021,7 @@ public void testRemoteConnectionInfoBwComp() throws IOException {
 
     public void testRenderConnectionInfoXContent() throws IOException {
         RemoteConnectionInfo stats = new RemoteConnectionInfo("test_cluster",
-            Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS,1)),
+            Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)),
             4, 3, TimeValue.timeValueMinutes(30), true);
         stats = assertSerialization(stats);
         XContentBuilder builder = XContentFactory.jsonBuilder();
@@ -1016,10 +1030,10 @@ public void testRenderConnectionInfoXContent() throws IOException {
         builder.endObject();
         assertEquals("{\"test_cluster\":{\"seeds\":[\"0.0.0.0:1\"],\"connected\":true," +
             "\"num_nodes_connected\":3,\"max_connections_per_cluster\":4,\"initial_connect_timeout\":\"30m\"," +
-                "\"skip_unavailable\":true}}", Strings.toString(builder));
+            "\"skip_unavailable\":true}}", Strings.toString(builder));
 
         stats = new RemoteConnectionInfo("some_other_cluster",
-            Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS,1), new TransportAddress(TransportAddress.META_ADDRESS,2)),
+            Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1), new TransportAddress(TransportAddress.META_ADDRESS, 2)),
             2, 0, TimeValue.timeValueSeconds(30), false);
         stats = assertSerialization(stats);
         builder = XContentFactory.jsonBuilder();
@@ -1027,14 +1041,14 @@ public void testRenderConnectionInfoXContent() throws IOException {
         stats.toXContent(builder, null);
         builder.endObject();
         assertEquals("{\"some_other_cluster\":{\"seeds\":[\"0.0.0.0:1\",\"0.0.0.0:2\"],"
-                + "\"connected\":false,\"num_nodes_connected\":0,\"max_connections_per_cluster\":2,\"initial_connect_timeout\":\"30s\"," +
-                "\"skip_unavailable\":false}}", Strings.toString(builder));
+            + "\"connected\":false,\"num_nodes_connected\":0,\"max_connections_per_cluster\":2,\"initial_connect_timeout\":\"30s\"," +
+            "\"skip_unavailable\":false}}", Strings.toString(builder));
     }
 
     public void testEnsureConnected() throws IOException, InterruptedException {
         List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
         try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
-            MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
             DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
             DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
             knownNodes.add(seedTransport.getLocalDiscoNode());
@@ -1129,7 +1143,7 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted
         try {
             final int numDiscoverableNodes = randomIntBetween(5, 20);
             List<DiscoveryNode> discoverableNodes = new ArrayList<>(numDiscoverableNodes);
-            for (int i = 0; i < numDiscoverableNodes; i++ ) {
+            for (int i = 0; i < numDiscoverableNodes; i++) {
                 MockTransportService transportService = startTransport("discoverable_node" + i, knownNodes, Version.CURRENT);
                 discoverableNodes.add(transportService.getLocalDiscoNode());
                 discoverableTransports.add(transportService);
@@ -1214,12 +1228,12 @@ public void testClusterNameIsChecked() throws Exception {
 
         Settings settings = Settings.builder().put("cluster.name", "testClusterNameIsChecked").build();
         try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT, threadPool, settings);
-            MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT, threadPool,
-                settings);
-            MockTransportService otherClusterTransport = startTransport("other_cluster_discoverable_node", otherClusterKnownNodes,
-                    Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build());
-            MockTransportService otherClusterDiscoverable= startTransport("other_cluster_discoverable_node", otherClusterKnownNodes,
-                    Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build())) {
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT, threadPool,
+                 settings);
+             MockTransportService otherClusterTransport = startTransport("other_cluster_discoverable_node", otherClusterKnownNodes,
+                 Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build());
+             MockTransportService otherClusterDiscoverable = startTransport("other_cluster_discoverable_node", otherClusterKnownNodes,
+                 Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build())) {
             DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
             DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
             knownNodes.add(seedTransport.getLocalDiscoNode());
@@ -1279,33 +1293,39 @@ public DiscoveryNode getNode() {
 
                     @Override
                     public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
-                            throws TransportException {
+                        throws TransportException {
                         // no-op
                     }
 
                     @Override
-                    public void close() {
+                    public void addCloseListener(ActionListener<Void> listener) {
                         // no-op
                     }
-                };
-                service.addDelegate(connectedNode.getAddress(), new MockTransportService.DelegateTransport(service.getOriginalTransport()) {
+
                     @Override
-                    public Connection getConnection(DiscoveryNode node) {
-                        if (node == connectedNode) {
-                            return seedConnection;
-                        }
-                        return super.getConnection(node);
+                    public boolean isClosed() {
+                        return false;
                     }
 
                     @Override
-                    public boolean nodeConnected(DiscoveryNode node) {
-                        return node.equals(connectedNode);
+                    public void close() {
+                        // no-op
+                    }
+                };
+
+                service.addNodeConnectedBehavior(connectedNode.getAddress(), (connectionManager, discoveryNode)
+                    -> discoveryNode.equals(connectedNode));
+
+                service.addGetConnectionBehavior(connectedNode.getAddress(), (connectionManager, discoveryNode) -> {
+                    if (discoveryNode == connectedNode) {
+                        return seedConnection;
                     }
+                    return connectionManager.getConnection(discoveryNode);
                 });
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                        Collections.singletonList(connectedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    Collections.singletonList(connectedNode), service, Integer.MAX_VALUE, n -> true)) {
                     connection.addConnectedNode(connectedNode);
                     for (int i = 0; i < 10; i++) {
                         //always a direct connection as the remote node is already connected
diff --git a/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java b/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java
index d16300bf266d6..8474b9947129e 100644
--- a/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java
@@ -198,7 +198,7 @@ protected void stopInternal() {
                 }
 
                 @Override
-                public NodeChannels getConnection(DiscoveryNode node) {
+                public NodeChannels openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) {
                     int numConnections = MockTcpTransport.LIGHT_PROFILE.getNumConnections();
                     ArrayList<TcpChannel> fakeChannels = new ArrayList<>(numConnections);
                     for (int i = 0; i < numConnections; ++i) {
@@ -209,7 +209,7 @@ public NodeChannels getConnection(DiscoveryNode node) {
             };
 
             DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT);
-            Transport.Connection connection = transport.getConnection(node);
+            Transport.Connection connection = transport.openConnection(node, null);
             connection.sendRequest(42, "foobar", request, TransportRequestOptions.EMPTY);
 
             BytesReference reference = messageCaptor.get();
diff --git a/server/src/test/java/org/elasticsearch/update/UpdateIT.java b/server/src/test/java/org/elasticsearch/update/UpdateIT.java
index e4ea078b8f716..85ebf01ef28c0 100644
--- a/server/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/server/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -93,6 +93,7 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
                 }
 
                 Map<String, Object> source = (Map<String, Object>) ctx.get("_source");
+                params.remove("ctx");
                 source.putAll(params);
 
                 return ctx;
diff --git a/test/framework/src/main/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverter.java b/test/framework/src/main/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverter.java
new file mode 100644
index 0000000000000..35ad331f532ff
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverter.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.logging;
+
+import java.util.Arrays;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.pattern.ConverterKeys;
+import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
+import org.apache.logging.log4j.core.pattern.PatternConverter;
+import org.elasticsearch.test.ESIntegTestCase;
+
+/**
+ * Converts {@code %test_thread_info} in log4j patterns into information
+ * based on the loggin thread's name. If that thread is part of an
+ * {@link ESIntegTestCase} then this information is the node name.
+ */
+@Plugin(category = PatternConverter.CATEGORY, name = "TestInfoPatternConverter")
+@ConverterKeys({"test_thread_info"})
+public class TestThreadInfoPatternConverter extends LogEventPatternConverter {
+    /**
+     * Called by log4j2 to initialize this converter.
+     */
+    public static TestThreadInfoPatternConverter newInstance(final String[] options) {
+        if (options.length > 0) {
+            throw new IllegalArgumentException("no options supported but options provided: "
+                    + Arrays.toString(options));
+        }
+        return new TestThreadInfoPatternConverter();
+    }
+
+    private TestThreadInfoPatternConverter() {
+        super("TestInfo", "test_thread_info");
+    }
+
+    @Override
+    public void format(LogEvent event, StringBuilder toAppendTo) {
+        toAppendTo.append(threadInfo(event.getThreadName()));
+    }
+
+    private static final Pattern ELASTICSEARCH_THREAD_NAME_PATTERN =
+            Pattern.compile("elasticsearch\\[(.+)\\]\\[.+\\].+");
+    private static final Pattern TEST_THREAD_NAME_PATTERN =
+            Pattern.compile("TEST-.+\\.(.+)-seed#\\[.+\\]");
+    private static final Pattern TEST_SUITE_INIT_THREAD_NAME_PATTERN =
+            Pattern.compile("SUITE-.+-worker");
+    private static final Pattern NOT_YET_NAMED_NODE_THREAD_NAME_PATTERN =
+            Pattern.compile("test_SUITE-CHILD_VM.+cluster\\[T#(.+)\\]");
+    static String threadInfo(String threadName) {
+        Matcher m = ELASTICSEARCH_THREAD_NAME_PATTERN.matcher(threadName);
+        if (m.matches()) {
+            // Thread looks like a node thread so use the node name
+            return m.group(1);
+        }
+        m = TEST_THREAD_NAME_PATTERN.matcher(threadName);
+        if (m.matches()) {
+            /*
+             * Thread looks like a test thread so use the test method name.
+             * It'd be pretty reasonable not to use a prefix at all here but
+             * the logger layout pretty much expects one and the test method
+             * can be pretty nice to have around anyway.
+             */
+            return m.group(1);
+        }
+        m = TEST_SUITE_INIT_THREAD_NAME_PATTERN.matcher(threadName);
+        if (m.matches()) {
+            /*
+             * Thread looks like test suite initialization or tead down and
+             * we don't have any more information to give. Like above, we
+             * could spit out nothing here but the logger layout expect
+             * something and it *is* nice to know what lines come from test
+             * teardown and initialization.
+             */
+            return "suite";
+        }
+        m = NOT_YET_NAMED_NODE_THREAD_NAME_PATTERN.matcher(threadName);
+        if (m.matches()) {
+            /*
+             * These are as yet unnamed integ test nodes. I'd prefer to log
+             * the node name but I don't have it yet.
+             */
+            return "integ_" + m.group(1) + "";
+        }
+        /*
+         * These are uncategorized threads. We log the entire thread name in
+         * case it is useful. We wrap it in `[]` so you tell that it is a
+         * thread name rather than a node name or something.
+         */
+        return "[" + threadName + "]";
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
index f26522245493f..2a84a8f424616 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
@@ -460,7 +460,7 @@ public void onFailedEngine(String reason, @Nullable Exception e) {
                 TimeValue.timeValueMinutes(5), refreshListenerList, Collections.emptyList(), indexSort, handler,
                 new NoneCircuitBreakerService(),
                 globalCheckpointSupplier == null ?
-                    new ReplicationTracker(shardId, allocationId.getId(), indexSettings, SequenceNumbers.NO_OPS_PERFORMED) :
+                    new ReplicationTracker(shardId, allocationId.getId(), indexSettings, SequenceNumbers.NO_OPS_PERFORMED, update -> {}) :
                     globalCheckpointSupplier, primaryTerm::get);
         return config;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index 2b1841c39ae52..77bc644909ab8 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -84,6 +84,7 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -233,7 +234,7 @@ public void startPrimary() throws IOException {
             activeIds.add(primary.routingEntry().allocationId().getId());
             ShardRouting startedRoutingEntry = ShardRoutingHelper.moveToStarted(primary.routingEntry());
             IndexShardRoutingTable routingTable = routingTable(shr -> shr == primary.routingEntry() ? startedRoutingEntry : shr);
-            primary.updateShardState(startedRoutingEntry, primary.getPrimaryTerm(), null,
+            primary.updateShardState(startedRoutingEntry, primary.getPendingPrimaryTerm(), null,
                 currentClusterStateVersion.incrementAndGet(), activeIds, routingTable, Collections.emptySet());
             for (final IndexShard replica : replicas) {
                 recoverReplica(replica);
@@ -279,20 +280,10 @@ public synchronized List<IndexShard> getReplicas() {
         /**
          * promotes the specific replica as the new primary
          */
-        public synchronized Future<PrimaryReplicaSyncer.ResyncTask> promoteReplicaToPrimary(IndexShard replica) throws IOException {
-            final long newTerm = indexMetaData.primaryTerm(shardId.id()) + 1;
-            IndexMetaData.Builder newMetaData = IndexMetaData.builder(indexMetaData).primaryTerm(shardId.id(), newTerm);
-            indexMetaData = newMetaData.build();
-            assertTrue(replicas.remove(replica));
-            closeShards(primary);
-            primary = replica;
-            assert primary.routingEntry().active() : "only active replicas can be promoted to primary: " + primary.routingEntry();
+        public Future<PrimaryReplicaSyncer.ResyncTask> promoteReplicaToPrimary(IndexShard replica) throws IOException {
             PlainActionFuture<PrimaryReplicaSyncer.ResyncTask> fut = new PlainActionFuture<>();
-            ShardRouting primaryRouting = replica.routingEntry().moveActiveReplicaToPrimary();
-            IndexShardRoutingTable routingTable = routingTable(shr -> shr == replica.routingEntry() ? primaryRouting : shr);
-
-            primary.updateShardState(primaryRouting,
-                newTerm, (shard, listener) -> primaryReplicaSyncer.resync(shard,
+            promoteReplicaToPrimary(replica,
+                (shard, listener) -> primaryReplicaSyncer.resync(shard,
                     new ActionListener<PrimaryReplicaSyncer.ResyncTask>() {
                         @Override
                         public void onResponse(PrimaryReplicaSyncer.ResyncTask resyncTask) {
@@ -305,11 +296,27 @@ public void onFailure(Exception e) {
                             listener.onFailure(e);
                             fut.onFailure(e);
                         }
-                    }), currentClusterStateVersion.incrementAndGet(), activeIds(), routingTable, Collections.emptySet());
-
+                    }));
             return fut;
         }
 
+        public synchronized void promoteReplicaToPrimary(IndexShard replica,
+                 BiConsumer<IndexShard, ActionListener<PrimaryReplicaSyncer.ResyncTask>> primaryReplicaSyncer)
+            throws IOException {
+            final long newTerm = indexMetaData.primaryTerm(shardId.id()) + 1;
+            IndexMetaData.Builder newMetaData = IndexMetaData.builder(indexMetaData).primaryTerm(shardId.id(), newTerm);
+            indexMetaData = newMetaData.build();
+            assertTrue(replicas.remove(replica));
+            closeShards(primary);
+            primary = replica;
+            assert primary.routingEntry().active() : "only active replicas can be promoted to primary: " + primary.routingEntry();
+            ShardRouting primaryRouting = replica.routingEntry().moveActiveReplicaToPrimary();
+            IndexShardRoutingTable routingTable = routingTable(shr -> shr == replica.routingEntry() ? primaryRouting : shr);
+
+            primary.updateShardState(primaryRouting, newTerm, primaryReplicaSyncer, currentClusterStateVersion.incrementAndGet(),
+                activeIds(), routingTable, Collections.emptySet());
+        }
+
         private synchronized Set<String> activeIds() {
             return shardRoutings().stream()
                 .filter(ShardRouting::active).map(ShardRouting::allocationId).map(AllocationId::getId).collect(Collectors.toSet());
@@ -425,7 +432,8 @@ public void syncGlobalCheckpoint() {
 
         private void updateAllocationIDsOnPrimary() throws IOException {
 
-            primary.updateShardState(primary.routingEntry(), primary.getPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(),
+            primary.updateShardState(primary.routingEntry(), primary.getPendingPrimaryTerm(), null,
+                currentClusterStateVersion.incrementAndGet(),
                 activeIds(), routingTable(Function.identity()), Collections.emptySet());
         }
     }
@@ -527,7 +535,7 @@ public void performOn(
                 IndexShard replica = replicationGroup.replicas.stream()
                         .filter(s -> replicaRouting.isSameAllocation(s.routingEntry())).findFirst().get();
                 replica.acquireReplicaOperationPermit(
-                        replicationGroup.primary.getPrimaryTerm(),
+                        replicationGroup.primary.getPendingPrimaryTerm(),
                         globalCheckpoint,
                         new ActionListener<Releasable>() {
                             @Override
@@ -605,7 +613,7 @@ protected PrimaryResult performOnPrimary(IndexShard primary, BulkShardRequest re
 
         @Override
         protected void performOnReplica(BulkShardRequest request, IndexShard replica) throws Exception {
-            executeShardBulkOnReplica(request, replica, getPrimaryShard().getPrimaryTerm(), getPrimaryShard().getGlobalCheckpoint());
+            executeShardBulkOnReplica(request, replica, getPrimaryShard().getPendingPrimaryTerm(), getPrimaryShard().getGlobalCheckpoint());
         }
     }
 
@@ -621,7 +629,8 @@ private TransportWriteAction.WritePrimaryResult<BulkShardRequest, BulkShardRespo
         final TransportWriteAction.WritePrimaryResult<BulkShardRequest, BulkShardResponse> result;
         try (Releasable ignored = permitAcquiredFuture.actionGet()) {
             MappingUpdatePerformer noopMappingUpdater = (update, shardId, type) -> { };
-            result = TransportShardBulkAction.performOnPrimary(request, primary, null, System::currentTimeMillis, noopMappingUpdater);
+            result = TransportShardBulkAction.performOnPrimary(request, primary, null, System::currentTimeMillis, noopMappingUpdater,
+                null);
         }
         TransportWriteActionTestHelper.performPostWriteActions(primary, request, result.location, logger);
         return result;
@@ -664,14 +673,18 @@ BulkShardRequest deleteOnPrimary(DeleteRequest request, IndexShard primary) thro
      * indexes the given requests on the supplied replica shard
      */
     void indexOnReplica(BulkShardRequest request, ReplicationGroup group, IndexShard replica) throws Exception {
-        executeShardBulkOnReplica(request, replica, group.primary.getPrimaryTerm(), group.primary.getGlobalCheckpoint());
+        indexOnReplica(request, group, replica, group.primary.getPendingPrimaryTerm());
+    }
+
+    void indexOnReplica(BulkShardRequest request, ReplicationGroup group, IndexShard replica, long term) throws Exception {
+        executeShardBulkOnReplica(request, replica, term, group.primary.getGlobalCheckpoint());
     }
 
     /**
      * Executes the delete request on the given replica shard.
      */
     void deleteOnReplica(BulkShardRequest request, ReplicationGroup group, IndexShard replica) throws Exception {
-        executeShardBulkOnReplica(request, replica, group.primary.getPrimaryTerm(), group.primary.getGlobalCheckpoint());
+        executeShardBulkOnReplica(request, replica, group.primary.getPendingPrimaryTerm(), group.primary.getGlobalCheckpoint());
     }
 
     class GlobalCheckpointSync extends ReplicationAction<
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index f9289f658614c..d2a84589669a6 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -425,7 +425,7 @@ public static void updateRoutingEntry(IndexShard shard, ShardRouting shardRoutin
         IndexShardRoutingTable newRoutingTable = new IndexShardRoutingTable.Builder(shardRouting.shardId())
             .addShard(shardRouting)
             .build();
-        shard.updateShardState(shardRouting, shard.getPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(),
+        shard.updateShardState(shardRouting, shard.getPendingPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(),
             inSyncIds, newRoutingTable, Collections.emptySet());
     }
 
@@ -514,8 +514,8 @@ protected final void recoverUnstartedReplica(final IndexShard replica,
             request,
             (int) ByteSizeUnit.MB.toBytes(1),
             Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), pNode.getName()).build());
-        primary.updateShardState(primary.routingEntry(), primary.getPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(),
-            inSyncIds, routingTable, Collections.emptySet());
+        primary.updateShardState(primary.routingEntry(), primary.getPendingPrimaryTerm(), null,
+            currentClusterStateVersion.incrementAndGet(), inSyncIds, routingTable, Collections.emptySet());
         recovery.recoverToTarget();
         recoveryTarget.markAsDone();
     }
@@ -536,9 +536,9 @@ protected void startReplicaAfterRecovery(IndexShard replica, IndexShard primary,
         Set<String> inSyncIdsWithReplica = new HashSet<>(inSyncIds);
         inSyncIdsWithReplica.add(replica.routingEntry().allocationId().getId());
         // update both primary and replica shard state
-        primary.updateShardState(primary.routingEntry(), primary.getPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(),
-            inSyncIdsWithReplica, newRoutingTable, Collections.emptySet());
-        replica.updateShardState(replica.routingEntry().moveToStarted(), replica.getPrimaryTerm(), null,
+        primary.updateShardState(primary.routingEntry(), primary.getPendingPrimaryTerm(), null,
+            currentClusterStateVersion.incrementAndGet(), inSyncIdsWithReplica, newRoutingTable, Collections.emptySet());
+        replica.updateShardState(replica.routingEntry().moveToStarted(), replica.getPendingPrimaryTerm(), null,
             currentClusterStateVersion.get(), inSyncIdsWithReplica, newRoutingTable, Collections.emptySet());
     }
 
@@ -560,7 +560,7 @@ protected void promoteReplica(IndexShard replica, Set<String> inSyncIds, IndexSh
             .removeShard(replica.routingEntry())
             .addShard(routingEntry)
             .build();
-        replica.updateShardState(routingEntry, replica.getPrimaryTerm() + 1,
+        replica.updateShardState(routingEntry, replica.getPendingPrimaryTerm() + 1,
             (is, listener) ->
                 listener.onResponse(new PrimaryReplicaSyncer.ResyncTask(1, "type", "action", "desc", null, Collections.emptyMap())),
             currentClusterStateVersion.incrementAndGet(),
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index 8e40e4bcf1468..0d340a91d4cea 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -96,6 +96,47 @@ public void execute(Map<String, Object> ctx) {
                 }
             };
             return context.factoryClazz.cast(factory);
+        } else if (context.instanceClazz.equals(UpdateScript.class)) {
+            UpdateScript.Factory factory = parameters -> new UpdateScript(parameters) {
+                @Override
+                public void execute(Map<String, Object> ctx) {
+                    final Map<String, Object> vars = new HashMap<>();
+                    vars.put("ctx", ctx);
+                    vars.put("params", parameters);
+                    vars.putAll(parameters);
+                    script.apply(vars);
+                }
+            };
+            return context.factoryClazz.cast(factory);
+        } else if (context.instanceClazz.equals(BucketAggregationScript.class)) {
+            BucketAggregationScript.Factory factory = parameters -> new BucketAggregationScript(parameters) {
+                @Override
+                public Double execute() {
+                    Object ret = script.apply(getParams());
+                    if (ret == null) {
+                        return null;
+                    } else {
+                        return ((Number) ret).doubleValue();
+                    }
+                }
+            };
+            return context.factoryClazz.cast(factory);
+        } else if (context.instanceClazz.equals(BucketAggregationSelectorScript.class)) {
+            BucketAggregationSelectorScript.Factory factory = parameters -> new BucketAggregationSelectorScript(parameters) {
+                @Override
+                public boolean execute() {
+                    return (boolean) script.apply(getParams());
+                }
+            };
+            return context.factoryClazz.cast(factory);
+        } else if (context.instanceClazz.equals(SignificantTermsHeuristicScoreScript.class)) {
+            SignificantTermsHeuristicScoreScript.Factory factory = () -> new SignificantTermsHeuristicScoreScript() {
+                @Override
+                public double execute(Map<String, Object> vars) {
+                    return ((Number) script.apply(vars)).doubleValue();
+                }
+            };
+            return context.factoryClazz.cast(factory);
         } else if (context.instanceClazz.equals(TemplateScript.class)) {
             TemplateScript.Factory factory = vars -> {
                 // TODO: need a better way to implement all these new contexts
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
index fccec5f784f8b..0a11325311dd8 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.test;
 
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.SeedUtils;
+
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.Version;
@@ -84,6 +87,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.function.Function;
 import java.util.stream.Stream;
@@ -124,14 +128,12 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
         ALIAS_TO_CONCRETE_FIELD_NAME.put(GEO_POINT_ALIAS_FIELD_NAME, GEO_POINT_FIELD_NAME);
     }
 
-    protected static Version indexVersionCreated;
-
     private static ServiceHolder serviceHolder;
     private static ServiceHolder serviceHolderWithNoType;
     private static int queryNameId = 0;
     private static Settings nodeSettings;
     private static Index index;
-    private static Index indexWithNoType;
+    private static long nowInMillis;
 
     protected static Index getIndex() {
         return index;
@@ -152,7 +154,7 @@ public static void beforeClass() {
             .build();
 
         index = new Index(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLength(10));
-        indexWithNoType = new Index(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLength(10));
+        nowInMillis = randomNonNegativeLong();
     }
 
     @Override
@@ -173,15 +175,19 @@ protected static String createUniqueRandomName() {
         return queryName;
     }
 
-    protected Settings indexSettings() {
+    protected Settings createTestIndexSettings() {
         // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
-        indexVersionCreated = randomBoolean() ? Version.CURRENT
+        Version indexVersionCreated = randomBoolean() ? Version.CURRENT
                 : VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.CURRENT);
         return Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated)
             .build();
     }
 
+    protected static IndexSettings indexSettings() {
+        return serviceHolder.idxSettings;
+    }
+
     protected static String expectedFieldName(String builderFieldName) {
         return ALIAS_TO_CONCRETE_FIELD_NAME.getOrDefault(builderFieldName, builderFieldName);
     }
@@ -195,14 +201,24 @@ public static void afterClass() throws Exception {
     }
 
     @Before
-    public void beforeTest() throws IOException {
+    public void beforeTest() throws Exception {
         if (serviceHolder == null) {
-            serviceHolder = new ServiceHolder(nodeSettings, indexSettings(), getPlugins(), this, true);
+            assert serviceHolderWithNoType == null;
+            // we initialize the serviceHolder and serviceHolderWithNoType just once, but need some
+            // calls to the randomness source during its setup. In order to not mix these calls with
+            // the randomness source that is later used in the test method, we use the master seed during
+            // this setup
+            long masterSeed = SeedUtils.parseSeed(RandomizedTest.getContext().getRunnerSeedAsString());
+            RandomizedTest.getContext().runWithPrivateRandomness(masterSeed, (Callable<Void>) () -> {
+                serviceHolder = new ServiceHolder(nodeSettings, createTestIndexSettings(), getPlugins(), nowInMillis,
+                        AbstractBuilderTestCase.this, true);
+                serviceHolderWithNoType = new ServiceHolder(nodeSettings, createTestIndexSettings(), getPlugins(), nowInMillis,
+                        AbstractBuilderTestCase.this, false);
+                return null;
+            });
         }
+
         serviceHolder.clientInvocationHandler.delegate = this;
-        if (serviceHolderWithNoType == null) {
-            serviceHolderWithNoType = new ServiceHolder(nodeSettings, indexSettings(), getPlugins(), this, false);
-        }
         serviceHolderWithNoType.clientInvocationHandler.delegate = this;
     }
 
@@ -305,13 +321,15 @@ private static class ServiceHolder implements Closeable {
         private final BitsetFilterCache bitsetFilterCache;
         private final ScriptService scriptService;
         private final Client client;
-        private final long nowInMillis = randomNonNegativeLong();
+        private final long nowInMillis;
 
         ServiceHolder(Settings nodeSettings,
                         Settings indexSettings,
                         Collection<Class<? extends Plugin>> plugins,
+                        long nowInMillis,
                         AbstractBuilderTestCase testCase,
                         boolean registerType) throws IOException {
+            this.nowInMillis = nowInMillis;
             Environment env = InternalSettingsPreparer.prepareEnvironment(nodeSettings);
             PluginsService pluginsService;
             pluginsService = new PluginsService(nodeSettings, null, env.modulesFile(), env.pluginsFile(), plugins);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
index 8c4076e327d70..fff29dff3f680 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
@@ -132,7 +132,7 @@ public static ClusterService createClusterService(ThreadPool threadPool, Discove
 
     public static ClusterService createClusterService(ThreadPool threadPool, DiscoveryNode localNode, ClusterSettings clusterSettings) {
         ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "ClusterServiceTests").build(),
-            clusterSettings, threadPool, Collections.emptyMap());
+            clusterSettings, threadPool);
         clusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
             @Override
             public void connectToNodes(DiscoveryNodes discoveryNodes) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index ee4f2f3b222e3..1498a69cf6a42 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -65,6 +65,7 @@
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -200,6 +201,8 @@ public static void resetPortCounter() {
         System.setProperty("log4j.shutdownHookEnabled", "false");
         System.setProperty("log4j2.disable.jmx", "true");
 
+        LogConfigurator.loadLog4jPlugins();
+
         // Enable Netty leak detection and monitor logger for logged leak errors
         System.setProperty("io.netty.leakDetection.level", "paranoid");
         String leakLoggerName = "io.netty.util.ResourceLeakDetector";
@@ -321,7 +324,7 @@ public static void restoreContentType() {
 
     @Before
     public final void before()  {
-        logger.info("[{}]: before test", getTestName());
+        logger.info("{}before test", getTestParamsForLogging());
         assertNull("Thread context initialized twice", threadContext);
         if (enableWarningsCheck()) {
             this.threadContext = new ThreadContext(Settings.EMPTY);
@@ -350,7 +353,16 @@ public final void after() throws Exception {
         }
         ensureAllSearchContextsReleased();
         ensureCheckIndexPassed();
-        logger.info("[{}]: after test", getTestName());
+        logger.info("{}after test", getTestParamsForLogging());
+    }
+
+    private String getTestParamsForLogging() {
+        String name = getTestName();
+        int start = name.indexOf('{');
+        if (start < 0) return "";
+        int end = name.lastIndexOf('}');
+        if (end < 0) return "";
+        return "[" + name.substring(start + 1, end) + "] ";
     }
 
     private void ensureNoWarnings() throws IOException {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index a0468091c847e..5b4f528913f18 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -2008,8 +2008,10 @@ public void ensureEstimatedStats() {
                 final CircuitBreakerService breakerService = getInstanceFromNode(CircuitBreakerService.class, nodeAndClient.node);
                 CircuitBreaker fdBreaker = breakerService.getBreaker(CircuitBreaker.FIELDDATA);
                 assertThat("Fielddata breaker not reset to 0 on node: " + name, fdBreaker.getUsed(), equalTo(0L));
-                CircuitBreaker acctBreaker = breakerService.getBreaker(CircuitBreaker.ACCOUNTING);
-                assertThat("Accounting breaker not reset to 0 on node: " + name, acctBreaker.getUsed(), equalTo(0L));
+                // TODO: This is commented out while Lee looks into the failures
+                // See: https://github.com/elastic/elasticsearch/issues/30290
+                // CircuitBreaker acctBreaker = breakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+                // assertThat("Accounting breaker not reset to 0 on node: " + name, acctBreaker.getUsed(), equalTo(0L));
                 // Anything that uses transport or HTTP can increase the
                 // request breaker (because they use bigarrays), because of
                 // that the breaker can sometimes be incremented from ping
diff --git a/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
index 06eefb7ccba14..68434f0f29ecc 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
@@ -48,7 +48,7 @@
 import java.util.Random;
 
 import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween;
-import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiOfLength;
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiLettersOfLength;
 import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomUnicodeOfLengthBetween;
 import static java.util.Collections.singleton;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_UUID_NA_VALUE;
@@ -122,7 +122,7 @@ public static Tuple<List<Object>, List<Object>> randomStoredFieldValues(Random r
                     expectedParsedValues.add(randomBoolean);
                     break;
                 case 7:
-                    String randomString = random.nextBoolean() ? RandomStrings.randomAsciiOfLengthBetween(random, 3, 10 ) :
+                    String randomString = random.nextBoolean() ? RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10) :
                             randomUnicodeOfLengthBetween(random, 3, 10);
                     originalValues.add(randomString);
                     expectedParsedValues.add(randomString);
@@ -191,11 +191,11 @@ private static void addFields(Random random, XContentBuilder builder, int minNum
         for (int i = 0; i < numFields; i++) {
             if (currentDepth < 5 && random.nextInt(100) >= 70) {
                 if (random.nextBoolean()) {
-                    builder.startObject(RandomStrings.randomAsciiOfLengthBetween(random, 6, 10));
+                    builder.startObject(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10));
                     addFields(random, builder, minNumFields, currentDepth + 1);
                     builder.endObject();
                 } else {
-                    builder.startArray(RandomStrings.randomAsciiOfLengthBetween(random, 6, 10));
+                    builder.startArray(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10));
                     int numElements = randomIntBetween(random, 1, 5);
                     boolean object = random.nextBoolean();
                     int dataType = -1;
@@ -214,7 +214,7 @@ private static void addFields(Random random, XContentBuilder builder, int minNum
                     builder.endArray();
                 }
             } else {
-                builder.field(RandomStrings.randomAsciiOfLengthBetween(random, 6, 10),
+                builder.field(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10),
                         randomFieldValue(random, randomDataType(random)));
             }
         }
@@ -227,9 +227,9 @@ private static int randomDataType(Random random) {
     private static Object randomFieldValue(Random random, int dataType) {
         switch(dataType) {
             case 0:
-                return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
+                return RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10);
             case 1:
-                return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
+                return RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10);
             case 2:
                 return random.nextLong();
             case 3:
@@ -287,10 +287,10 @@ public static Tuple<ShardInfo, ShardInfo> randomShardInfo(Random random, boolean
      * @param random Random generator
      */
     private static Tuple<Failure, Failure> randomShardInfoFailure(Random random) {
-        String index = randomAsciiOfLength(random, 5);
-        String indexUuid = randomAsciiOfLength(random, 5);
+        String index = randomAsciiLettersOfLength(random, 5);
+        String indexUuid = randomAsciiLettersOfLength(random, 5);
         int shardId = randomIntBetween(random, 1, 10);
-        String nodeId = randomAsciiOfLength(random, 5);
+        String nodeId = randomAsciiLettersOfLength(random, 5);
         RestStatus status = randomFrom(random, RestStatus.INTERNAL_SERVER_ERROR, RestStatus.FORBIDDEN, RestStatus.NOT_FOUND);
         boolean primary = random.nextBoolean();
         ShardId shard = new ShardId(index, indexUuid, shardId);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
index 877a6f2e98a67..aeffd26e214d6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
@@ -150,6 +150,10 @@ public ReproduceErrorMessageBuilder appendESProperties() {
             appendOpt("tests.locale", Locale.getDefault().toLanguageTag());
             appendOpt("tests.timezone", TimeZone.getDefault().getID());
             appendOpt("tests.distribution", System.getProperty("tests.distribution"));
+            appendOpt("compiler.java", System.getProperty("compiler.java"));
+            appendOpt("runtime.java", System.getProperty("runtime.java"));
+            appendOpt("javax.net.ssl.keyStorePassword", System.getProperty("javax.net.ssl.keyStorePassword"));
+            appendOpt("javax.net.ssl.trustStorePassword", System.getProperty("javax.net.ssl.trustStorePassword"));
             return this;
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index e25e8b43d3a72..ecb965040f87b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -148,7 +148,7 @@ public void initClient() throws IOException {
                 }
                 String host = stringUrl.substring(0, portSeparator);
                 int port = Integer.valueOf(stringUrl.substring(portSeparator + 1));
-                hosts.add(new HttpHost(host, port, getProtocol()));
+                hosts.add(buildHttpHost(host, port));
             }
             clusterHosts = unmodifiableList(hosts);
             logger.info("initializing REST clients against {}", clusterHosts);
@@ -160,6 +160,13 @@ public void initClient() throws IOException {
         assert clusterHosts != null;
     }
 
+    /**
+     * Construct a HttpHost from the given host and port
+     */
+    protected HttpHost buildHttpHost(String host, int port) {
+        return new HttpHost(host, port, getProtocol());
+    }
+
     /**
      * Clean up after the test case.
      */
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java
index ddd5837663521..b0b84cd2c2140 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java
@@ -28,7 +28,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
-import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 
 import java.io.IOException;
@@ -50,8 +50,8 @@ public ClientYamlDocsTestClient(
             final List<HttpHost> hosts,
             final Version esVersion,
             final Version masterVersion,
-            final CheckedConsumer<RestClientBuilder, IOException> clientBuilderConsumer) {
-        super(restSpec, restClient, hosts, esVersion, masterVersion, clientBuilderConsumer);
+            final CheckedSupplier<RestClientBuilder, IOException> clientBuilderWithSniffedNodes) {
+        super(restSpec, restClient, hosts, esVersion, masterVersion, clientBuilderWithSniffedNodes);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
index 2d6bcc8cf5665..856fd2a32ded4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
@@ -26,7 +26,6 @@
 import org.apache.http.util.EntityUtils;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
-import org.elasticsearch.client.Node;
 import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
@@ -34,7 +33,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
-import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestPath;
@@ -66,7 +65,7 @@ public class ClientYamlTestClient implements Closeable {
     private final Map<NodeSelector, RestClient> restClients = new HashMap<>();
     private final Version esVersion;
     private final Version masterVersion;
-    private final CheckedConsumer<RestClientBuilder, IOException> clientBuilderConsumer;
+    private final CheckedSupplier<RestClientBuilder, IOException> clientBuilderWithSniffedNodes;
 
     ClientYamlTestClient(
             final ClientYamlSuiteRestSpec restSpec,
@@ -74,13 +73,13 @@ public class ClientYamlTestClient implements Closeable {
             final List<HttpHost> hosts,
             final Version esVersion,
             final Version masterVersion,
-            final CheckedConsumer<RestClientBuilder, IOException> clientBuilderConsumer) {
+            final CheckedSupplier<RestClientBuilder, IOException> clientBuilderWithSniffedNodes) {
         assert hosts.size() > 0;
         this.restSpec = restSpec;
         this.restClients.put(NodeSelector.ANY, restClient);
         this.esVersion = esVersion;
         this.masterVersion = masterVersion;
-        this.clientBuilderConsumer = clientBuilderConsumer;
+        this.clientBuilderWithSniffedNodes = clientBuilderWithSniffedNodes;
     }
 
     public Version getEsVersion() {
@@ -199,10 +198,9 @@ public ClientYamlTestResponse callApi(String apiName, Map<String, String> params
     protected RestClient getRestClient(NodeSelector nodeSelector) {
         //lazily build a new client in case we need to point to some specific node
         return restClients.computeIfAbsent(nodeSelector, selector -> {
-            RestClient anyClient = restClients.get(NodeSelector.ANY);
-            RestClientBuilder builder = RestClient.builder(anyClient.getNodes().toArray(new Node[0]));
+            RestClientBuilder builder;
             try {
-                clientBuilderConsumer.accept(builder);
+                builder = clientBuilderWithSniffedNodes.get();
             } catch (IOException e) {
                 throw new UncheckedIOException(e);
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
index b97b4e8f6dace..91456d395d601 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.client.sniff.ElasticsearchNodesSniffer;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
@@ -58,13 +59,6 @@
 /**
  * Runs a suite of yaml tests shared with all the official Elasticsearch
  * clients against against an elasticsearch cluster.
- * <p>
- * <strong>IMPORTANT</strong>: These tests sniff the cluster for metadata
- * and hosts on startup and replace the list of hosts that they are
- * configured to use with the list sniffed from the cluster. So you can't
- * control which nodes receive the request by providing the right list of
- * nodes in the <code>tests.rest.cluster</code> system property. Instead
- * the tests must explictly use `node_selector`s.
  */
 public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
 
@@ -123,11 +117,6 @@ public static void initializeUseDefaultNumberOfShards() {
     @Before
     public void initAndResetContext() throws Exception {
         if (restTestExecutionContext == null) {
-            // Sniff host metadata in case we need it in the yaml tests
-            List<Node> nodesWithMetadata = sniffHostMetadata();
-            client().setNodes(nodesWithMetadata);
-            adminClient().setNodes(nodesWithMetadata);
-
             assert adminExecutionContext == null;
             assert blacklistPathMatchers == null;
             final ClientYamlSuiteRestSpec restSpec = ClientYamlSuiteRestSpec.load(SPEC_PATH);
@@ -166,8 +155,7 @@ protected ClientYamlTestClient initClientYamlTestClient(
             final List<HttpHost> hosts,
             final Version esVersion,
             final Version masterVersion) {
-        return new ClientYamlTestClient(restSpec, restClient, hosts, esVersion, masterVersion,
-                restClientBuilder -> configureClient(restClientBuilder, restClientSettings()));
+        return new ClientYamlTestClient(restSpec, restClient, hosts, esVersion, masterVersion, this::getClientBuilderWithSniffedHosts);
     }
 
     @AfterClass
@@ -408,13 +396,16 @@ protected boolean randomizeContentType() {
     }
 
     /**
-     * Sniff the cluster for host metadata.
+     * Sniff the cluster for host metadata and return a
+     * {@link RestClientBuilder} for a client with that metadata.
      */
-    private List<Node> sniffHostMetadata() throws IOException {
+    protected final RestClientBuilder getClientBuilderWithSniffedHosts() throws IOException {
         ElasticsearchNodesSniffer.Scheme scheme =
             ElasticsearchNodesSniffer.Scheme.valueOf(getProtocol().toUpperCase(Locale.ROOT));
         ElasticsearchNodesSniffer sniffer = new ElasticsearchNodesSniffer(
                 adminClient(), ElasticsearchNodesSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT, scheme);
-        return sniffer.sniff();
+        RestClientBuilder builder = RestClient.builder(sniffer.sniff().toArray(new Node[0]));
+        configureClient(builder, restClientSettings());
+        return builder;
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
index 4e46a9ec89fd1..e1346d3f6967d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
@@ -391,7 +391,32 @@ private static NodeSelector parseAttributeValuesSelector(XContentParser parser)
             if (token == XContentParser.Token.FIELD_NAME) {
                 key = parser.currentName();
             } else if (token.isValue()) {
-                NodeSelector newSelector = new HasAttributeNodeSelector(key, parser.text());
+                /*
+                 * HasAttributeNodeSelector selects nodes that do not have
+                 * attribute metadata set so it can be used against nodes that
+                 * have not yet been sniffed. In these tests we expect the node
+                 * metadata to be explicitly sniffed if we need it and we'd
+                 * like to hard fail if it is not so we wrap the selector so we
+                 * can assert that the data is sniffed.
+                 */
+                NodeSelector delegate = new HasAttributeNodeSelector(key, parser.text());
+                NodeSelector newSelector = new NodeSelector() {
+                    @Override
+                    public void select(Iterable<Node> nodes) {
+                        for (Node node : nodes) {
+                            if (node.getAttributes() == null) {
+                                throw new IllegalStateException("expected [attributes] metadata to be set but got "
+                                        + node);
+                            }
+                        }
+                        delegate.select(nodes);
+                    }
+
+                    @Override
+                    public String toString() {
+                        return delegate.toString();
+                    }
+                };
                 result = result == NodeSelector.ANY ?
                     newSelector : new ComposeNodeSelector(result, newSelector);
             } else {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
index ffdf79c0636b2..a7d3e85d3009d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
@@ -20,18 +20,22 @@
 package org.elasticsearch.test.transport;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ConnectionManager;
 import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.RemoteTransportException;
 import org.elasticsearch.transport.RequestHandlerRegistry;
@@ -39,13 +43,14 @@
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportConnectionListener;
 import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportStats;
 
 import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -53,9 +58,11 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.function.Function;
 
 import static org.apache.lucene.util.LuceneTestCase.rarely;
 
@@ -84,6 +91,45 @@ public CapturedRequest(DiscoveryNode node, long requestId, String action, Transp
     private ConcurrentMap<Long, Tuple<DiscoveryNode, String>> requests = new ConcurrentHashMap<>();
     private BlockingQueue<CapturedRequest> capturedRequests = ConcurrentCollections.newBlockingQueue();
 
+    public TransportService createCapturingTransportService(Settings settings, ThreadPool threadPool, TransportInterceptor interceptor,
+                                                            Function<BoundTransportAddress, DiscoveryNode> localNodeFactory,
+                                                            @Nullable ClusterSettings clusterSettings, Set<String> taskHeaders) {
+        StubbableConnectionManager connectionManager = new StubbableConnectionManager(new ConnectionManager(settings, this, threadPool),
+            settings, this, threadPool);
+        connectionManager.setDefaultNodeConnectedBehavior((cm, discoveryNode) -> true);
+        connectionManager.setDefaultConnectBehavior((cm, discoveryNode) -> new Connection() {
+            @Override
+            public DiscoveryNode getNode() {
+                return discoveryNode;
+            }
+
+            @Override
+            public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+                throws TransportException {
+                requests.put(requestId, Tuple.tuple(discoveryNode, action));
+                capturedRequests.add(new CapturedRequest(discoveryNode, requestId, action, request));
+            }
+
+            @Override
+            public void addCloseListener(ActionListener<Void> listener) {
+
+            }
+
+            @Override
+            public boolean isClosed() {
+                return false;
+            }
+
+            @Override
+            public void close() {
+
+            }
+        });
+        return new TransportService(settings, this, threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders,
+            connectionManager);
+
+    }
+
     /** returns all requests captured so far. Doesn't clear the captured request list. See {@link #clear()} */
     public CapturedRequest[] capturedRequests() {
         return capturedRequests.toArray(new CapturedRequest[0]);
@@ -195,7 +241,7 @@ public void handleError(final long requestId, final TransportException e) {
     }
 
     @Override
-    public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
+    public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) {
         return new Connection() {
             @Override
             public DiscoveryNode getNode() {
@@ -204,13 +250,23 @@ public DiscoveryNode getNode() {
 
             @Override
             public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
-                throws IOException, TransportException {
+                throws TransportException {
                 requests.put(requestId, Tuple.tuple(node, action));
                 capturedRequests.add(new CapturedRequest(node, requestId, action, request));
             }
 
             @Override
-            public void close() throws IOException {
+            public void addCloseListener(ActionListener<Void> listener) {
+
+            }
+
+            @Override
+            public boolean isClosed() {
+                return false;
+            }
+
+            @Override
+            public void close() {
 
             }
         };
@@ -236,23 +292,6 @@ public TransportAddress[] addressesFromString(String address, int perAddressLimi
         return new TransportAddress[0];
     }
 
-    @Override
-    public boolean nodeConnected(DiscoveryNode node) {
-        return true;
-    }
-
-    @Override
-    public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
-                              CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
-        throws ConnectTransportException {
-
-    }
-
-    @Override
-    public void disconnectFromNode(DiscoveryNode node) {
-
-    }
-
     @Override
     public Lifecycle.State lifecycleState() {
         return null;
@@ -282,14 +321,6 @@ public List<String> getLocalAddresses() {
         return Collections.emptyList();
     }
 
-    public Connection getConnection(DiscoveryNode node) {
-        try {
-            return openConnection(node, null);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
     @Override
     public <Request extends TransportRequest> void registerRequestHandler(RequestHandlerRegistry<Request> reg) {
         synchronized (requestHandlerMutex) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 7f818de29d430..40a6ad6476d4a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -20,15 +20,12 @@
 package org.elasticsearch.test.transport;
 
 import com.carrotsearch.randomizedtesting.SysGlobals;
-
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.component.Lifecycle;
-import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
@@ -40,7 +37,6 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
@@ -48,21 +44,18 @@
 import org.elasticsearch.test.tasks.MockTaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionManager;
 import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.MockTcpTransport;
 import org.elasticsearch.transport.RequestHandlerRegistry;
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportConnectionListener;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.transport.TransportStats;
 
 import java.io.IOException;
-import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -71,18 +64,18 @@
 import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
-import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 /**
- * A mock transport service that allows to simulate different network topology failures.
+ * A mock delegate service that allows to simulate different network topology failures.
  * Internally it maps TransportAddress objects to rules that inject failures.
  * Adding rules for a node is done by adding rules for all bound addresses of a node
  * (and the publish address, if different).
- * Matching requests to rules is based on the transport address associated with the
+ * Matching requests to rules is based on the delegate address associated with the
  * discovery node of the request, namely by DiscoveryNode.getAddress().
  * This address is usually the publish address of the node but can also be a different one
  * (for example, @see org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing, which constructs
@@ -147,9 +140,15 @@ public MockTransportService(Settings settings, Transport transport, ThreadPool t
     public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor,
                                 Function<BoundTransportAddress, DiscoveryNode> localNodeFactory,
                                 @Nullable ClusterSettings clusterSettings, Set<String> taskHeaders) {
-        super(settings, new LookupTestTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings,
-            taskHeaders);
-        this.original = transport;
+        this(settings, new StubbableTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders);
+    }
+
+    private MockTransportService(Settings settings, StubbableTransport transport, ThreadPool threadPool, TransportInterceptor interceptor,
+                                 Function<BoundTransportAddress, DiscoveryNode> localNodeFactory,
+                                 @Nullable ClusterSettings clusterSettings, Set<String> taskHeaders) {
+        super(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders,
+            new StubbableConnectionManager(new ConnectionManager(settings, transport, threadPool), settings, transport, threadPool));
+        this.original = transport.getDelegate();
     }
 
     public static TransportAddress[] extractTransportAddresses(TransportService transportService) {
@@ -173,11 +172,12 @@ protected TaskManager createTaskManager(Settings settings, ThreadPool threadPool
      * Clears all the registered rules.
      */
     public void clearAllRules() {
-        transport().transports.clear();
+        transport().clearBehaviors();
+        connectionManager().clearBehaviors();
     }
 
     /**
-     * Clears the rule associated with the provided transport service.
+     * Clears the rule associated with the provided delegate service.
      */
     public void clearRule(TransportService transportService) {
         for (TransportAddress transportAddress : extractTransportAddresses(transportService)) {
@@ -186,20 +186,11 @@ public void clearRule(TransportService transportService) {
     }
 
     /**
-     * Clears the rule associated with the provided transport address.
+     * Clears the rule associated with the provided delegate address.
      */
     public void clearRule(TransportAddress transportAddress) {
-        Transport transport = transport().transports.remove(transportAddress);
-        if (transport instanceof ClearableTransport) {
-            ((ClearableTransport) transport).clearRule();
-        }
-    }
-
-    /**
-     * Returns the original Transport service wrapped by this mock transport service.
-     */
-    public Transport original() {
-        return original;
+        transport().clearBehavior(transportAddress);
+        connectionManager().clearBehavior(transportAddress);
     }
 
     /**
@@ -217,30 +208,14 @@ public void addFailToSendNoConnectRule(TransportService transportService) {
      * is added to fail as well.
      */
     public void addFailToSendNoConnectRule(TransportAddress transportAddress) {
-        addDelegate(transportAddress, new DelegateTransport(original) {
-
-            @Override
-            public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
-                                      CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
-                throws ConnectTransportException {
-                if (original.nodeConnected(node) == false) {
-                    // connecting to an already connected node is a no-op
-                    throw new ConnectTransportException(node, "DISCONNECT: simulated");
-                }
-            }
-
-            @Override
-            public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
-                throw new ConnectTransportException(node, "DISCONNECT: simulated");
-            }
+        transport().addConnectBehavior(transportAddress, (transport, discoveryNode, profile) -> {
+            throw new ConnectTransportException(discoveryNode, "DISCONNECT: simulated");
+        });
 
-            @Override
-            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
-                                       TransportRequestOptions options) throws IOException {
-                connection.close();
-                // send the request, which will blow up
-                connection.sendRequest(requestId, action, request, options);
-            }
+        transport().addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> {
+            connection.close();
+            // send the request, which will blow up
+            connection.sendRequest(requestId, action, request, options);
         });
     }
 
@@ -271,18 +246,12 @@ public void addFailToSendNoConnectRule(TransportService transportService, final
      * Adds a rule that will cause matching operations to throw ConnectTransportExceptions
      */
     public void addFailToSendNoConnectRule(TransportAddress transportAddress, final Set<String> blockedActions) {
-
-        addDelegate(transportAddress, new DelegateTransport(original) {
-
-            @Override
-            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
-                                       TransportRequestOptions options) throws IOException {
-                if (blockedActions.contains(action)) {
-                    logger.info("--> preventing {} request", action);
-                    connection.close();
-                }
-                connection.sendRequest(requestId, action, request, options);
+        transport().addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> {
+            if (blockedActions.contains(action)) {
+                logger.info("--> preventing {} request", action);
+                connection.close();
             }
+            connection.sendRequest(requestId, action, request, options);
         });
     }
 
@@ -301,28 +270,12 @@ public void addUnresponsiveRule(TransportService transportService) {
      * and failing to connect once the rule was added.
      */
     public void addUnresponsiveRule(TransportAddress transportAddress) {
-        addDelegate(transportAddress, new DelegateTransport(original) {
-
-            @Override
-            public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
-                                      CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
-                throws ConnectTransportException {
-                if (original.nodeConnected(node) == false) {
-                    // connecting to an already connected node is a no-op
-                    throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
-                }
-            }
-
-            @Override
-            public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
-                throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
-            }
+        transport().addConnectBehavior(transportAddress, (transport, discoveryNode, profile) -> {
+            throw new ConnectTransportException(discoveryNode, "UNRESPONSIVE: simulated");
+        });
 
-            @Override
-            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
-                                       TransportRequestOptions options) throws IOException {
-                // don't send anything, the receiving node is unresponsive
-            }
+        transport().addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> {
+            // don't send anything, the receiving node is unresponsive
         });
     }
 
@@ -347,70 +300,38 @@ public void addUnresponsiveRule(TransportService transportService, final TimeVal
     public void addUnresponsiveRule(TransportAddress transportAddress, final TimeValue duration) {
         final long startTime = System.currentTimeMillis();
 
-        addDelegate(transportAddress, new ClearableTransport(original) {
-            private final Queue<Runnable> requestsToSendWhenCleared = new LinkedBlockingDeque<>();
-            private boolean cleared = false;
+        Supplier<TimeValue> delaySupplier = () -> new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime));
 
-            TimeValue getDelay() {
-                return new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime));
+        transport().addConnectBehavior(transportAddress, (transport, discoveryNode, profile) -> {
+            TimeValue delay = delaySupplier.get();
+            if (delay.millis() <= 0) {
+                return original.openConnection(discoveryNode, profile);
             }
 
-            @Override
-            public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
-                                      CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
-                throws ConnectTransportException {
-                if (original.nodeConnected(node)) {
-                    // connecting to an already connected node is a no-op
-                    return;
-                }
-                TimeValue delay = getDelay();
-                if (delay.millis() <= 0) {
-                    original.connectToNode(node, connectionProfile, connectionValidator);
-                    return;
-                }
-
-                // TODO: Replace with proper setting
-                TimeValue connectingTimeout = TcpTransport.TCP_CONNECT_TIMEOUT.getDefault(Settings.EMPTY);
-                try {
-                    if (delay.millis() < connectingTimeout.millis()) {
-                        Thread.sleep(delay.millis());
-                        original.connectToNode(node, connectionProfile, connectionValidator);
-                    } else {
-                        Thread.sleep(connectingTimeout.millis());
-                        throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
-                    }
-                } catch (InterruptedException e) {
-                    throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
+            // TODO: Replace with proper setting
+            TimeValue connectingTimeout = TcpTransport.TCP_CONNECT_TIMEOUT.getDefault(Settings.EMPTY);
+            try {
+                if (delay.millis() < connectingTimeout.millis()) {
+                    Thread.sleep(delay.millis());
+                    return original.openConnection(discoveryNode, profile);
+                } else {
+                    Thread.sleep(connectingTimeout.millis());
+                    throw new ConnectTransportException(discoveryNode, "UNRESPONSIVE: simulated");
                 }
+            } catch (InterruptedException e) {
+                throw new ConnectTransportException(discoveryNode, "UNRESPONSIVE: simulated");
             }
+        });
 
-            @Override
-            public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
-                TimeValue delay = getDelay();
-                if (delay.millis() <= 0) {
-                    return original.openConnection(node, profile);
-                }
-
-                // TODO: Replace with proper setting
-                TimeValue connectingTimeout = TcpTransport.TCP_CONNECT_TIMEOUT.getDefault(Settings.EMPTY);
-                try {
-                    if (delay.millis() < connectingTimeout.millis()) {
-                        Thread.sleep(delay.millis());
-                        return original.openConnection(node, profile);
-                    } else {
-                        Thread.sleep(connectingTimeout.millis());
-                        throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
-                    }
-                } catch (InterruptedException e) {
-                    throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
-                }
-            }
+        transport().addSendBehavior(transportAddress, new StubbableTransport.SendRequestBehavior() {
+            private final Queue<Runnable> requestsToSendWhenCleared = new LinkedBlockingDeque<>();
+            private boolean cleared = false;
 
             @Override
-            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
-                                       TransportRequestOptions options) throws IOException {
+            public void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request,
+                                    TransportRequestOptions options) throws IOException {
                 // delayed sending - even if larger then the request timeout to simulated a potential late response from target node
-                TimeValue delay = getDelay();
+                TimeValue delay = delaySupplier.get();
                 if (delay.millis() <= 0) {
                     connection.sendRequest(requestId, action, request, options);
                     return;
@@ -450,7 +371,7 @@ protected void doRun() throws IOException {
             }
 
             @Override
-            public void clearRule() {
+            public void clearCallback() {
                 synchronized (this) {
                     assert cleared == false;
                     cleared = true;
@@ -461,234 +382,128 @@ public void clearRule() {
     }
 
     /**
-     * Adds a new delegate transport that is used for communication with the given transport service.
+     * Adds a new send behavior that is used for communication with the given delegate service.
      *
-     * @return {@code true} iff no other delegate was registered for any of the addresses bound by transport service.
+     * @return {@code true} if no other send behavior was registered for any of the addresses bound by delegate service.
      */
-    public boolean addDelegate(TransportService transportService, DelegateTransport transport) {
+    public boolean addSendBehavior(TransportService transportService, StubbableTransport.SendRequestBehavior sendBehavior) {
         boolean noRegistered = true;
         for (TransportAddress transportAddress : extractTransportAddresses(transportService)) {
-            noRegistered &= addDelegate(transportAddress, transport);
+            noRegistered &= addSendBehavior(transportAddress, sendBehavior);
         }
         return noRegistered;
     }
 
     /**
-     * Adds a new delegate transport that is used for communication with the given transport address.
+     * Adds a new send behavior that is used for communication with the given delegate address.
      *
-     * @return {@code true} iff no other delegate was registered for this address before.
+     * @return {@code true} if no other send behavior was registered for this address before.
      */
-    public boolean addDelegate(TransportAddress transportAddress, DelegateTransport transport) {
-        return transport().transports.put(transportAddress, transport) == null;
-    }
-
-    private LookupTestTransport transport() {
-        return (LookupTestTransport) transport;
+    public boolean addSendBehavior(TransportAddress transportAddress, StubbableTransport.SendRequestBehavior sendBehavior) {
+        return transport().addSendBehavior(transportAddress, sendBehavior);
     }
 
     /**
-     * A lookup transport that has a list of potential Transport implementations to delegate to for node operations,
-     * if none is registered, then the default one is used.
+     * Adds a send behavior that is the default send behavior.
+     *
+     * @return {@code true} if no default send behavior was registered
      */
-    private static class LookupTestTransport extends DelegateTransport {
-
-        final ConcurrentMap<TransportAddress, Transport> transports = ConcurrentCollections.newConcurrentMap();
-
-        LookupTestTransport(Transport transport) {
-            super(transport);
-        }
-
-        private Transport getTransport(DiscoveryNode node) {
-            Transport transport = transports.get(node.getAddress());
-            if (transport != null) {
-                return transport;
-            }
-            return this.transport;
-        }
-
-        @Override
-        public boolean nodeConnected(DiscoveryNode node) {
-            return getTransport(node).nodeConnected(node);
-        }
-
-        @Override
-        public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
-                                  CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
-            throws ConnectTransportException {
-            getTransport(node).connectToNode(node, connectionProfile, connectionValidator);
-        }
-
-        @Override
-        public void disconnectFromNode(DiscoveryNode node) {
-            getTransport(node).disconnectFromNode(node);
-        }
+    public boolean addSendBehavior(StubbableTransport.SendRequestBehavior behavior) {
+        return transport().setDefaultSendBehavior(behavior);
+    }
 
-        @Override
-        public Connection getConnection(DiscoveryNode node) {
-            return getTransport(node).getConnection(node);
-        }
 
-        @Override
-        public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
-            return getTransport(node).openConnection(node, profile);
+    /**
+     * Adds a new connect behavior that is used for creating connections with the given delegate service.
+     *
+     * @return {@code true} if no other send behavior was registered for any of the addresses bound by delegate service.
+     */
+    public boolean addConnectBehavior(TransportService transportService, StubbableTransport.OpenConnectionBehavior connectBehavior) {
+        boolean noRegistered = true;
+        for (TransportAddress transportAddress : extractTransportAddresses(transportService)) {
+            noRegistered &= addConnectBehavior(transportAddress, connectBehavior);
         }
+        return noRegistered;
     }
 
     /**
-     * A pure delegate transport.
-     * Can be extracted to a common class if needed in other places in the codebase.
+     * Adds a new connect behavior that is used for creating connections with the given delegate address.
+     *
+     * @return {@code true} if no other send behavior was registered for this address before.
      */
-    public static class DelegateTransport implements Transport {
-
-        protected final Transport transport;
-
-
-        public DelegateTransport(Transport transport) {
-            this.transport = transport;
-        }
-
-        @Override
-        public void addConnectionListener(TransportConnectionListener listener) {
-            transport.addConnectionListener(listener);
-        }
-
-        @Override
-        public boolean removeConnectionListener(TransportConnectionListener listener) {
-            return transport.removeConnectionListener(listener);
-        }
-
-        @Override
-        public <Request extends TransportRequest> void registerRequestHandler(RequestHandlerRegistry<Request> reg) {
-            transport.registerRequestHandler(reg);
-        }
-
-        @Override
-        public RequestHandlerRegistry getRequestHandler(String action) {
-            return transport.getRequestHandler(action);
-        }
-
-        @Override
-        public BoundTransportAddress boundAddress() {
-            return transport.boundAddress();
-        }
-
-        @Override
-        public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
-            return transport.addressesFromString(address, perAddressLimit);
-        }
-
-        @Override
-        public boolean nodeConnected(DiscoveryNode node) {
-            return transport.nodeConnected(node);
-        }
-
-        @Override
-        public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
-                                  CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator)
-            throws ConnectTransportException {
-            transport.connectToNode(node, connectionProfile, connectionValidator);
-        }
-
-        @Override
-        public void disconnectFromNode(DiscoveryNode node) {
-            transport.disconnectFromNode(node);
-        }
-
-        @Override
-        public List<String> getLocalAddresses() {
-            return transport.getLocalAddresses();
-        }
-
-        @Override
-        public Connection getConnection(DiscoveryNode node) {
-            return new FilteredConnection(transport.getConnection(node)) {
-                @Override
-                public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
-                    throws IOException, TransportException {
-                    DelegateTransport.this.sendRequest(connection, requestId, action, request, options);
-                }
-            };
-        }
-
-        @Override
-        public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
-            return new FilteredConnection(transport.openConnection(node, profile)) {
-                @Override
-                public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
-                    throws IOException, TransportException {
-                    DelegateTransport.this.sendRequest(connection, requestId, action, request, options);
-                }
-            };
-        }
-
-        @Override
-        public TransportStats getStats() {
-            return transport.getStats();
-        }
-
-        @Override
-        public ResponseHandlers getResponseHandlers() {
-            return transport.getResponseHandlers();
-        }
-
-        @Override
-        public Lifecycle.State lifecycleState() {
-            return transport.lifecycleState();
-        }
-
-        @Override
-        public void addLifecycleListener(LifecycleListener listener) {
-            transport.addLifecycleListener(listener);
-        }
-
-        @Override
-        public void removeLifecycleListener(LifecycleListener listener) {
-            transport.removeLifecycleListener(listener);
-        }
-
-        @Override
-        public void start() {
-            transport.start();
-        }
+    public boolean addConnectBehavior(TransportAddress transportAddress, StubbableTransport.OpenConnectionBehavior connectBehavior) {
+        return transport().addConnectBehavior(transportAddress, connectBehavior);
+    }
 
-        @Override
-        public void stop() {
-            transport.stop();
+    /**
+     * Adds a new get connection behavior that is used for communication with the given delegate service.
+     *
+     * @return {@code true} if no other get connection behavior was registered for any of the addresses bound by delegate service.
+     */
+    public boolean addGetConnectionBehavior(TransportService transportService, StubbableConnectionManager.GetConnectionBehavior behavior) {
+        boolean noRegistered = true;
+        for (TransportAddress transportAddress : extractTransportAddresses(transportService)) {
+            noRegistered &= addGetConnectionBehavior(transportAddress, behavior);
         }
+        return noRegistered;
+    }
 
-        @Override
-        public void close() {
-            transport.close();
-        }
+    /**
+     * Adds a get connection behavior that is used for communication with the given delegate address.
+     *
+     * @return {@code true} if no other get connection behavior was registered for this address before.
+     */
+    public boolean addGetConnectionBehavior(TransportAddress transportAddress, StubbableConnectionManager.GetConnectionBehavior behavior) {
+        return connectionManager().addConnectBehavior(transportAddress, behavior);
+    }
 
-        @Override
-        public Map<String, BoundTransportAddress> profileBoundAddresses() {
-            return transport.profileBoundAddresses();
-        }
+    /**
+     * Adds a get connection behavior that is the default get connection behavior.
+     *
+     * @return {@code true} if no default get connection behavior was registered.
+     */
+    public boolean addGetConnectionBehavior(StubbableConnectionManager.GetConnectionBehavior behavior) {
+        return connectionManager().setDefaultConnectBehavior(behavior);
+    }
 
-        protected void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request,
-                                   TransportRequestOptions options) throws IOException {
-            connection.sendRequest(requestId, action, request, options);
+    /**
+     * Adds a node connected behavior that is used for the given delegate service.
+     *
+     * @return {@code true} if no other node connected behavior was registered for any of the addresses bound by delegate service.
+     */
+    public boolean addNodeConnectedBehavior(TransportService transportService, StubbableConnectionManager.NodeConnectedBehavior behavior) {
+        boolean noRegistered = true;
+        for (TransportAddress transportAddress : extractTransportAddresses(transportService)) {
+            noRegistered &= addNodeConnectedBehavior(transportAddress, behavior);
         }
+        return noRegistered;
     }
 
     /**
-     * The delegate transport instances defined in this class mock various kinds of disruption types. This subclass adds a method
-     * {@link #clearRule()} so that when the disruptions are cleared (see {@link #clearRule(TransportService)}) this gives the
-     * disruption a possibility to run clean-up actions.
+     * Adds a node connected behavior that is used for the given delegate address.
+     *
+     * @return {@code true} if no other node connected behavior was registered for this address before.
      */
-    public abstract static class ClearableTransport extends DelegateTransport {
+    public boolean addNodeConnectedBehavior(TransportAddress transportAddress, StubbableConnectionManager.NodeConnectedBehavior behavior) {
+        return connectionManager().addNodeConnectedBehavior(transportAddress, behavior);
+    }
 
-        public ClearableTransport(Transport transport) {
-            super(transport);
-        }
+    /**
+     * Adds a node connected behavior that is the default node connected behavior.
+     *
+     * @return {@code true} if no default node connected behavior was registered.
+     */
+    public boolean addNodeConnectedBehavior(StubbableConnectionManager.NodeConnectedBehavior behavior) {
+        return connectionManager().setDefaultNodeConnectedBehavior(behavior);
+    }
 
-        /**
-         * Called by {@link #clearRule(TransportService)}
-         */
-        public abstract void clearRule();
+    public StubbableTransport transport() {
+        return (StubbableTransport) transport;
     }
 
+    public StubbableConnectionManager connectionManager() {
+        return (StubbableConnectionManager) connectionManager;
+    }
 
     List<Tracer> activeTracers = new CopyOnWriteArrayList<>();
 
@@ -766,78 +581,36 @@ protected void traceRequestSent(DiscoveryNode node, long requestId, String actio
         }
     }
 
-    private static class FilteredConnection implements Transport.Connection {
-        protected final Transport.Connection connection;
-
-        private FilteredConnection(Transport.Connection connection) {
-            this.connection = connection;
-        }
-
-        @Override
-        public DiscoveryNode getNode() {
-            return connection.getNode();
-        }
-
-        @Override
-        public Version getVersion() {
-            return connection.getVersion();
-        }
-
-        @Override
-        public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
-            throws IOException, TransportException {
-            connection.sendRequest(requestId, action, request, options);
-        }
-
-        @Override
-        public void close() throws IOException {
-            connection.close();
-        }
-
-        @Override
-        public Object getCacheKey() {
-            return connection.getCacheKey();
-        }
-    }
-
     public Transport getOriginalTransport() {
         Transport transport = transport();
-        while (transport instanceof DelegateTransport) {
-            transport = ((DelegateTransport) transport).transport;
+        while (transport instanceof StubbableTransport) {
+            transport = ((StubbableTransport) transport).getDelegate();
         }
         return transport;
     }
 
     @Override
     public Transport.Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
-        FilteredConnection filteredConnection = new FilteredConnection(super.openConnection(node, profile)) {
-            final AtomicBoolean closed = new AtomicBoolean(false);
-
-            @Override
-            public void close() throws IOException {
-                try {
-                    super.close();
-                } finally {
-                    if (closed.compareAndSet(false, true)) {
-                        synchronized (openConnections) {
-                            List<Transport.Connection> connections = openConnections.get(node);
-                            boolean remove = connections.remove(this);
-                            assert remove;
-                            if (connections.isEmpty()) {
-                                openConnections.remove(node);
-                            }
-                        }
-                    }
-                }
+        Transport.Connection connection = super.openConnection(node, profile);
 
-            }
-        };
         synchronized (openConnections) {
             List<Transport.Connection> connections = openConnections.computeIfAbsent(node,
                 (n) -> new CopyOnWriteArrayList<>());
-            connections.add(filteredConnection);
+            connections.add(connection);
         }
-        return filteredConnection;
+
+        connection.addCloseListener(ActionListener.wrap(() -> {
+            synchronized (openConnections) {
+                List<Transport.Connection> connections = openConnections.get(node);
+                boolean remove = connections.remove(connection);
+                assert remove : "Should have removed connection";
+                if (connections.isEmpty()) {
+                    openConnections.remove(node);
+                }
+            }
+        }));
+
+        return connection;
     }
 
     @Override
@@ -851,4 +624,5 @@ protected void doClose() throws IOException {
     public DiscoveryNode getLocalDiscoNode() {
         return this.getLocalNode();
     }
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java
new file mode 100644
index 0000000000000..a74cb2752c284
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test.transport;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionManager;
+import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportConnectionListener;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+public class StubbableConnectionManager extends ConnectionManager {
+
+    private final ConnectionManager delegate;
+    private final ConcurrentMap<TransportAddress, GetConnectionBehavior> getConnectionBehaviors;
+    private final ConcurrentMap<TransportAddress, NodeConnectedBehavior> nodeConnectedBehaviors;
+    private volatile GetConnectionBehavior defaultGetConnectionBehavior = ConnectionManager::getConnection;
+    private volatile NodeConnectedBehavior defaultNodeConnectedBehavior = ConnectionManager::nodeConnected;
+
+    public StubbableConnectionManager(ConnectionManager delegate, Settings settings, Transport transport, ThreadPool threadPool) {
+        super(settings, transport, threadPool);
+        this.delegate = delegate;
+        this.getConnectionBehaviors = new ConcurrentHashMap<>();
+        this.nodeConnectedBehaviors = new ConcurrentHashMap<>();
+    }
+
+    public boolean addConnectBehavior(TransportAddress transportAddress, GetConnectionBehavior connectBehavior) {
+        return getConnectionBehaviors.put(transportAddress, connectBehavior) == null;
+    }
+
+    public boolean setDefaultConnectBehavior(GetConnectionBehavior behavior) {
+        GetConnectionBehavior prior = defaultGetConnectionBehavior;
+        defaultGetConnectionBehavior = behavior;
+        return prior == null;
+    }
+
+    public boolean addNodeConnectedBehavior(TransportAddress transportAddress, NodeConnectedBehavior behavior) {
+        return nodeConnectedBehaviors.put(transportAddress, behavior) == null;
+    }
+
+    public boolean setDefaultNodeConnectedBehavior(NodeConnectedBehavior behavior) {
+        NodeConnectedBehavior prior = defaultNodeConnectedBehavior;
+        defaultNodeConnectedBehavior = behavior;
+        return prior == null;
+    }
+
+    public void clearBehaviors() {
+        getConnectionBehaviors.clear();
+        nodeConnectedBehaviors.clear();
+    }
+
+    public void clearBehavior(TransportAddress transportAddress) {
+        getConnectionBehaviors.remove(transportAddress);
+        nodeConnectedBehaviors.remove(transportAddress);
+    }
+
+    @Override
+    public Transport.Connection getConnection(DiscoveryNode node) {
+        TransportAddress address = node.getAddress();
+        GetConnectionBehavior behavior = getConnectionBehaviors.getOrDefault(address, defaultGetConnectionBehavior);
+        return behavior.getConnection(delegate, node);
+    }
+
+    @Override
+    public boolean nodeConnected(DiscoveryNode node) {
+        TransportAddress address = node.getAddress();
+        NodeConnectedBehavior behavior = nodeConnectedBehaviors.getOrDefault(address, defaultNodeConnectedBehavior);
+        return behavior.nodeConnected(delegate, node);
+    }
+
+    @Override
+    public void addListener(TransportConnectionListener listener) {
+        delegate.addListener(listener);
+    }
+
+    @Override
+    public void removeListener(TransportConnectionListener listener) {
+        delegate.removeListener(listener);
+    }
+
+    @Override
+    public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
+                              CheckedBiConsumer<Transport.Connection, ConnectionProfile, IOException> connectionValidator)
+        throws ConnectTransportException {
+        delegate.connectToNode(node, connectionProfile, connectionValidator);
+    }
+
+    @Override
+    public void disconnectFromNode(DiscoveryNode node) {
+        delegate.disconnectFromNode(node);
+    }
+
+    @Override
+    public int connectedNodeCount() {
+        return delegate.connectedNodeCount();
+    }
+
+    @Override
+    public void close() {
+        delegate.close();
+    }
+
+    @FunctionalInterface
+    public interface GetConnectionBehavior {
+        Transport.Connection getConnection(ConnectionManager connectionManager, DiscoveryNode discoveryNode);
+    }
+
+    @FunctionalInterface
+    public interface NodeConnectedBehavior {
+        boolean nodeConnected(ConnectionManager connectionManager, DiscoveryNode discoveryNode);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java
new file mode 100644
index 0000000000000..5a0dd3b7f6d53
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.transport;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.component.Lifecycle;
+import org.elasticsearch.common.component.LifecycleListener;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.RequestHandlerRegistry;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportConnectionListener;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportStats;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class StubbableTransport implements Transport {
+
+    private final ConcurrentHashMap<TransportAddress, SendRequestBehavior> sendBehaviors = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<TransportAddress, OpenConnectionBehavior> connectBehaviors = new ConcurrentHashMap<>();
+    private volatile SendRequestBehavior defaultSendRequest = null;
+    private volatile OpenConnectionBehavior defaultConnectBehavior = null;
+    private final Transport delegate;
+
+
+    public StubbableTransport(Transport transport) {
+        this.delegate = transport;
+    }
+
+    boolean setDefaultSendBehavior(SendRequestBehavior sendBehavior) {
+        SendRequestBehavior prior = defaultSendRequest;
+        defaultSendRequest = sendBehavior;
+        return prior == null;
+    }
+
+    boolean addSendBehavior(TransportAddress transportAddress, SendRequestBehavior sendBehavior) {
+        return sendBehaviors.put(transportAddress, sendBehavior) == null;
+    }
+
+    boolean addConnectBehavior(TransportAddress transportAddress, OpenConnectionBehavior connectBehavior) {
+        return connectBehaviors.put(transportAddress, connectBehavior) == null;
+    }
+
+    void clearBehaviors() {
+        sendBehaviors.clear();
+        connectBehaviors.clear();
+    }
+
+    void clearBehavior(TransportAddress transportAddress) {
+        SendRequestBehavior behavior = sendBehaviors.remove(transportAddress);
+        if (behavior != null) {
+            behavior.clearCallback();
+        }
+        connectBehaviors.remove(transportAddress);
+    }
+
+    Transport getDelegate() {
+        return delegate;
+    }
+
+    @Override
+    public void addConnectionListener(TransportConnectionListener listener) {
+        delegate.addConnectionListener(listener);
+    }
+
+    @Override
+    public boolean removeConnectionListener(TransportConnectionListener listener) {
+        return delegate.removeConnectionListener(listener);
+    }
+
+    @Override
+    public <Request extends TransportRequest> void registerRequestHandler(RequestHandlerRegistry<Request> reg) {
+        delegate.registerRequestHandler(reg);
+    }
+
+    @Override
+    public RequestHandlerRegistry getRequestHandler(String action) {
+        return delegate.getRequestHandler(action);
+    }
+
+    @Override
+    public BoundTransportAddress boundAddress() {
+        return delegate.boundAddress();
+    }
+
+    @Override
+    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException {
+        return delegate.addressesFromString(address, perAddressLimit);
+    }
+
+    @Override
+    public List<String> getLocalAddresses() {
+        return delegate.getLocalAddresses();
+    }
+
+    @Override
+    public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) {
+        TransportAddress address = node.getAddress();
+        OpenConnectionBehavior behavior = connectBehaviors.getOrDefault(address, defaultConnectBehavior);
+        Connection connection;
+        if (behavior == null) {
+            connection = delegate.openConnection(node, profile);
+        } else {
+            connection = behavior.openConnection(delegate, node, profile);
+        }
+
+        return new WrappedConnection(connection);
+    }
+
+    @Override
+    public TransportStats getStats() {
+        return delegate.getStats();
+    }
+
+    @Override
+    public Transport.ResponseHandlers getResponseHandlers() {
+        return delegate.getResponseHandlers();
+    }
+
+    @Override
+    public Lifecycle.State lifecycleState() {
+        return delegate.lifecycleState();
+    }
+
+    @Override
+    public void addLifecycleListener(LifecycleListener listener) {
+        delegate.addLifecycleListener(listener);
+    }
+
+    @Override
+    public void removeLifecycleListener(LifecycleListener listener) {
+        delegate.removeLifecycleListener(listener);
+    }
+
+    @Override
+    public void start() {
+        delegate.start();
+    }
+
+    @Override
+    public void stop() {
+        delegate.stop();
+    }
+
+    @Override
+    public void close() {
+        delegate.close();
+    }
+
+    @Override
+    public Map<String, BoundTransportAddress> profileBoundAddresses() {
+        return delegate.profileBoundAddresses();
+    }
+
+    private class WrappedConnection implements Transport.Connection {
+
+        private final Transport.Connection connection;
+
+        private WrappedConnection(Transport.Connection connection) {
+            this.connection = connection;
+        }
+
+        @Override
+        public DiscoveryNode getNode() {
+            return connection.getNode();
+        }
+
+        @Override
+        public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+            throws IOException, TransportException {
+            TransportAddress address = connection.getNode().getAddress();
+            SendRequestBehavior behavior = sendBehaviors.getOrDefault(address, defaultSendRequest);
+            if (behavior == null) {
+                connection.sendRequest(requestId, action, request, options);
+            } else {
+                behavior.sendRequest(connection, requestId, action, request, options);
+            }
+        }
+
+        @Override
+        public boolean sendPing() {
+            return connection.sendPing();
+        }
+
+        @Override
+        public void addCloseListener(ActionListener<Void> listener) {
+            connection.addCloseListener(listener);
+        }
+
+
+        @Override
+        public boolean isClosed() {
+            return connection.isClosed();
+        }
+
+        @Override
+        public Version getVersion() {
+            return connection.getVersion();
+        }
+
+        @Override
+        public Object getCacheKey() {
+            return connection.getCacheKey();
+        }
+
+        @Override
+        public void close() {
+            connection.close();
+        }
+    }
+
+    @FunctionalInterface
+    public interface OpenConnectionBehavior {
+        Connection openConnection(Transport transport, DiscoveryNode discoveryNode, ConnectionProfile profile);
+    }
+
+    @FunctionalInterface
+    public interface SendRequestBehavior {
+        void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                         TransportRequestOptions options) throws IOException;
+
+        default void clearCallback() {
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java
index 3eca4818c4ab1..dc5305d951bb0 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java
@@ -40,9 +40,11 @@
 import org.elasticsearch.nio.NioSocketChannel;
 import org.elasticsearch.nio.ServerChannelContext;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.TcpChannel;
 import org.elasticsearch.transport.TcpServerChannel;
 import org.elasticsearch.transport.TcpTransport;
+import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.Transports;
 
 import java.io.IOException;
@@ -51,6 +53,8 @@
 import java.nio.ByteBuffer;
 import java.nio.channels.ServerSocketChannel;
 import java.nio.channels.SocketChannel;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
@@ -128,6 +132,34 @@ protected void stopInternal() {
         profileToChannelFactory.clear();
     }
 
+    @Override
+    protected ConnectionProfile resolveConnectionProfile(ConnectionProfile connectionProfile) {
+        ConnectionProfile resolvedProfile = resolveConnectionProfile(connectionProfile, defaultConnectionProfile);
+        if (resolvedProfile.getNumConnections() <= 3) {
+            return resolvedProfile;
+        }
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        Set<TransportRequestOptions.Type> allTypesWithConnection = new HashSet<>();
+        Set<TransportRequestOptions.Type> allTypesWithoutConnection = new HashSet<>();
+        for (TransportRequestOptions.Type type : TransportRequestOptions.Type.values()) {
+            int numConnections = resolvedProfile.getNumConnectionsPerType(type);
+            if (numConnections > 0) {
+                allTypesWithConnection.add(type);
+            } else {
+                allTypesWithoutConnection.add(type);
+            }
+        }
+
+        // make sure we maintain at least the types that are supported by this profile even if we only use a single channel for them.
+        builder.addConnections(3, allTypesWithConnection.toArray(new TransportRequestOptions.Type[0]));
+        if (allTypesWithoutConnection.isEmpty() == false) {
+            builder.addConnections(0, allTypesWithoutConnection.toArray(new TransportRequestOptions.Type[0]));
+        }
+        builder.setHandshakeTimeout(resolvedProfile.getHandshakeTimeout());
+        builder.setConnectTimeout(resolvedProfile.getConnectTimeout());
+        return builder.build();
+    }
+
     private void exceptionCaught(NioSocketChannel channel, Exception exception) {
         onException((TcpChannel) channel, exception);
     }
diff --git a/test/framework/src/main/resources/log4j2-test.properties b/test/framework/src/main/resources/log4j2-test.properties
index 828555ed52e6f..842c9c79d7963 100644
--- a/test/framework/src/main/resources/log4j2-test.properties
+++ b/test/framework/src/main/resources/log4j2-test.properties
@@ -1,7 +1,7 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%test_thread_info]%marker %m%n
 
 rootLogger.level = ${sys:tests.es.logger.level:-info}
 rootLogger.appenderRef.console.ref = console
diff --git a/test/framework/src/main/resources/log4j2.component.properties b/test/framework/src/main/resources/log4j2.component.properties
new file mode 100644
index 0000000000000..8ce0fb493b7b6
--- /dev/null
+++ b/test/framework/src/main/resources/log4j2.component.properties
@@ -0,0 +1,9 @@
+# RandomizedRunner renames threads and we'd like to look at accurate thread
+# names. Also, this is the default in log4j2's master branch for versions of
+# Java after 1.8_102 which most of the versions we use are anyway. After that
+# version of Java calling `Thread.getName()` doesn't allocate so UNCACHED
+# ought to be faster than CACHED because it doesn't have to deal with
+# ThreadLocals.
+# While we don't use AsyncLogger, we do end up with mutable log events and
+# those use this key for configuration.
+AsyncLogger.ThreadNameStrategy = UNCACHED
diff --git a/test/framework/src/test/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverterTests.java b/test/framework/src/test/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverterTests.java
new file mode 100644
index 0000000000000..8a98b867a46f5
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverterTests.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.logging;
+
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
+
+import static org.elasticsearch.common.logging.TestThreadInfoPatternConverter.threadInfo;
+
+public class TestThreadInfoPatternConverterTests extends ESTestCase {
+    private static String suiteInfo;
+
+    @BeforeClass
+    public static void captureSuiteInfo() {
+        suiteInfo = threadInfo(Thread.currentThread().getName());
+    }
+
+    public void testThreadInfo() {
+        // Threads that are part of a node get the node name
+        String nodeName = randomAlphaOfLength(5);
+        String threadName = EsExecutors.threadName(nodeName, randomAlphaOfLength(20))
+                + "[T#" + between(0, 1000) + "]";
+        assertEquals(nodeName, threadInfo(threadName));
+
+        // Test threads get the test name
+        assertEquals(getTestName(), threadInfo(Thread.currentThread().getName()));
+
+        // Suite initalization gets "suite"
+        assertEquals("suite", suiteInfo);
+
+        // And stuff that doesn't match anything gets wrapped in [] so we can see it
+        String unmatched = randomAlphaOfLength(5);
+        assertEquals("[" + unmatched + "]", threadInfo(unmatched));
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
index c4c96d9fe2b42..4c714c7e9aa14 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
@@ -540,6 +540,15 @@ public void testNodeSelectorByVersion() throws IOException {
         doSection.execute(context);
         verify(context).callApi("indices.get_field_mapping", singletonMap("index", "test_index"),
                 emptyList(), emptyMap(), doSection.getApiCallSection().getNodeSelector());
+
+        {
+            List<Node> badNodes = new ArrayList<>();
+            badNodes.add(new Node(new HttpHost("dummy")));
+            Exception e = expectThrows(IllegalStateException.class, () ->
+                    doSection.getApiCallSection().getNodeSelector().select(badNodes));
+            assertEquals("expected [version] metadata to be set but got [host=http://dummy]",
+                    e.getMessage());
+        }
     }
 
     private static Node nodeWithVersion(String version) {
@@ -568,6 +577,14 @@ public void testNodeSelectorByAttribute() throws IOException {
             doSection.getApiCallSection().getNodeSelector().select(nodes);
             assertEquals(Arrays.asList(hasAttr), nodes);
         }
+        {
+            List<Node> badNodes = new ArrayList<>();
+            badNodes.add(new Node(new HttpHost("dummy")));
+            Exception e = expectThrows(IllegalStateException.class, () ->
+                    doSection.getApiCallSection().getNodeSelector().select(badNodes));
+            assertEquals("expected [attributes] metadata to be set but got [host=http://dummy]",
+                    e.getMessage());
+        }
 
         parser = createParser(YamlXContent.yamlXContent,
                 "node_selector:\n" +
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java
index 108411dee5b89..3a78f366bd87f 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java
@@ -94,6 +94,11 @@ protected MockTransportService build(Settings settings, Version version, Cluster
         return transportService;
     }
 
+    @Override
+    protected int channelsPerNodeConnection() {
+        return 3;
+    }
+
     @Override
     protected void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException {
         TcpTransport.NodeChannels channels = (TcpTransport.NodeChannels) connection;
diff --git a/test/logger-usage/build.gradle b/test/logger-usage/build.gradle
index 98fe76bfcdc31..c16dab6a625c8 100644
--- a/test/logger-usage/build.gradle
+++ b/test/logger-usage/build.gradle
@@ -45,3 +45,13 @@ thirdPartyAudit.excludes = [
   'org.osgi.framework.wiring.BundleWire',
   'org.osgi.framework.wiring.BundleWiring'
 ]
+
+if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+  // Used by Log4J 2.11.1
+  thirdPartyAudit.excludes += [
+    'java.io.ObjectInputFilter',
+    'java.io.ObjectInputFilter$Config',
+    'java.io.ObjectInputFilter$FilterInfo',
+    'java.io.ObjectInputFilter$Status'
+  ]
+}
\ No newline at end of file
diff --git a/x-pack/docs/en/ml/detector-custom-rules.asciidoc b/x-pack/docs/en/ml/detector-custom-rules.asciidoc
index 18d516fae2de0..8513c7e4d2566 100644
--- a/x-pack/docs/en/ml/detector-custom-rules.asciidoc
+++ b/x-pack/docs/en/ml/detector-custom-rules.asciidoc
@@ -1,11 +1,11 @@
 [role="xpack"]
 [[ml-configuring-detector-custom-rules]]
-=== Customizing detectors with rules and filters
+=== Customizing detectors with custom rules
 
-<<ml-rules,Rules and filters>> enable you to change the behavior of anomaly 
+<<ml-rules,Custom rules>> enable you to change the behavior of anomaly
 detectors based on domain-specific knowledge.
 
-Rules describe _when_ a detector should take a certain _action_ instead
+Custom rules describe _when_ a detector should take a certain _action_ instead
 of following its default behavior. To specify the _when_ a rule uses
 a `scope` and `conditions`. You can think of `scope` as the categorical
 specification of a rule, while `conditions` are the numerical part.
@@ -14,9 +14,9 @@ scope and conditions.
 
 Let us see how those can be configured by examples.
 
-==== Specifying rule scope
+==== Specifying custom rule scope
 
-Let us assume we are configuring a job in order to DNS data exfiltration.
+Let us assume we are configuring a job in order to detect DNS data exfiltration.
 Our data contain fields "subdomain" and "highest_registered_domain".
 We can use a detector that looks like `high_info_content(subdomain) over highest_registered_domain`.
 If we run such a job it is possible that we discover a lot of anomalies on 
@@ -25,8 +25,8 @@ are not interested in such anomalies. Ideally, we could instruct the detector to
 skip results for domains that we consider safe. Using a rule with a scope allows 
 us to achieve this.
 
-First, we need to create a list with our safe domains. Those lists are called 
-`filters` in {ml}. Filters can be shared across jobs.
+First, we need to create a list of our safe domains. Those lists are called 
+_filters_ in {ml}. Filters can be shared across jobs.
 
 We create our filter using the {ref}/ml-put-filter.html[put filter API]:
 
@@ -40,8 +40,8 @@ PUT _xpack/ml/filters/safe_domains
 ----------------------------------
 // CONSOLE
 
-Now, we can create our job specifying a scope that uses the filter for the 
-`highest_registered_domain` field:
+Now, we can create our job specifying a scope that uses the `safe_domains` 
+filter for the `highest_registered_domain` field:
 
 [source,js]
 ----------------------------------
@@ -85,7 +85,9 @@ POST _xpack/ml/filters/safe_domains/_update
 // CONSOLE
 // TEST[setup:ml_filter_safe_domains]
 
-Note that we can provide scope for any of the partition/over/by fields.
+Note that we can use any of the `partition_field_name`, `over_field_name`, or 
+`by_field_name` fields in the `scope`.
+
 In the following example we scope multiple fields:
 
 [source,js]
@@ -125,7 +127,7 @@ PUT _xpack/ml/anomaly_detectors/scoping_multiple_fields
 Such a detector will skip results when the values of all 3 scoped fields
 are included in the referenced filters.
 
-==== Specifying rule conditions
+==== Specifying custom rule conditions
 
 Imagine a detector that looks for anomalies in CPU utilization.
 Given a machine that is idle for long enough, small movement in CPU could
@@ -204,17 +206,17 @@ PUT _xpack/ml/anomaly_detectors/rule_with_range
 ----------------------------------
 // CONSOLE
 
-==== Rules in the life-cycle of a job
+==== Custom rules in the life-cycle of a job
 
-Rules only affect results created after the rules were applied.
+Custom rules only affect results created after the rules were applied.
 Let us imagine that we have configured a job and it has been running
 for some time. After observing its results we decide that we can employ
 rules in order to get rid of some uninteresting results. We can use
-the update-job API to do so. However, the rule we added will only be in effect
-for any results created from the moment we added the rule onwards. Past results
-will remain unaffected.
+the {ref}/ml-update-job.html[update job API] to do so. However, the rule we 
+added will only be in effect for any results created from the moment we added 
+the rule onwards. Past results will remain unaffected.
 
-==== Using rules VS filtering data
+==== Using custom rules VS filtering data
 
 It might appear like using rules is just another way of filtering the data
 that feeds into a job. For example, a rule that skips results when the
diff --git a/x-pack/docs/en/rest-api/ml/filterresource.asciidoc b/x-pack/docs/en/rest-api/ml/filterresource.asciidoc
index 64768da4911c4..e942447c1ee60 100644
--- a/x-pack/docs/en/rest-api/ml/filterresource.asciidoc
+++ b/x-pack/docs/en/rest-api/ml/filterresource.asciidoc
@@ -13,4 +13,4 @@ A filter resource has the following properties:
 `items`::
   (array of strings) An array of strings which is the filter item list.
   
-For more information, see {stack-ov}/ml-rules.html[Machine learning rules and filters].
+For more information, see {stack-ov}/ml-rules.html[Machine learning custom rules].
diff --git a/x-pack/docs/en/rest-api/ml/get-filter.asciidoc b/x-pack/docs/en/rest-api/ml/get-filter.asciidoc
index 89f40cf331251..b4699e9d622cf 100644
--- a/x-pack/docs/en/rest-api/ml/get-filter.asciidoc
+++ b/x-pack/docs/en/rest-api/ml/get-filter.asciidoc
@@ -18,7 +18,7 @@ Retrieves filters.
 ===== Description
 
 You can get a single filter or all filters. For more information, see 
-{stack-ov}/ml-rules.html[Machine learning rules and filters].
+{stack-ov}/ml-rules.html[Machine learning custom rules].
 
 
 ==== Path Parameters
diff --git a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
index fb3424cab3fda..5b109b1c21d78 100644
--- a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
+++ b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
@@ -265,7 +265,7 @@ NOTE: The `field_name` cannot contain double quotes or backslashes.
   when there is no value for the by or partition fields. The default value is `false`.
 
 `custom_rules`::
-  (array) An array of rule objects, which enable customizing how the detector works.
+  (array) An array of custom rule objects, which enable customizing how the detector works.
   For example, a rule may dictate to the detector conditions under which results should be skipped.
   For more information see <<ml-detector-custom-rule,detector custom rule objects>>. +
 +
@@ -420,7 +420,7 @@ For more information, see
 {stack-ov}/ml-rules.html[Custom rules] enable you to customize the way detectors 
 operate. 
 
-A rule has the following properties:
+A custom rule has the following properties:
 
 `actions`::
   (array) The set of actions to be triggered when the rule applies.
@@ -455,7 +455,7 @@ A rule has the following properties:
 +
 --
 NOTE: If your detector uses `lat_long`, `metric`, `rare`, or `freq_rare` 
-functions, you cannot specify `conditions` for your rule.
+functions, you can only specify `conditions` that apply to `time`.
 
   
 A condition has the following properties: 
@@ -467,7 +467,7 @@ A condition has the following properties:
     (string) Specifies the condition operator. The available options are
     `gt` (greater than), `gte` (greater than or equals), `lt` (less than) and `lte` (less than or equals).
 `value`:::
-    (double) The value that is compared against the `applied_to` field using the `operator`.
+    (double) The value that is compared against the `applies_to` field using the `operator`.
 --
 
 A rule is required to either have a non-empty scope or at least one condition.
diff --git a/x-pack/docs/en/rest-api/rollup/get-job.asciidoc b/x-pack/docs/en/rest-api/rollup/get-job.asciidoc
index 7a7db9258b88a..96053dbfea64f 100644
--- a/x-pack/docs/en/rest-api/rollup/get-job.asciidoc
+++ b/x-pack/docs/en/rest-api/rollup/get-job.asciidoc
@@ -93,7 +93,8 @@ Which will yield the following response:
             "page_size" : 1000
           },
           "status" : {
-            "job_state" : "stopped"
+            "job_state" : "stopped",
+            "upgraded_doc_id": true
           },
           "stats" : {
             "pages_processed" : 0,
@@ -212,7 +213,8 @@ Which will yield the following response:
             "page_size" : 1000
           },
           "status" : {
-            "job_state" : "stopped"
+            "job_state" : "stopped",
+            "upgraded_doc_id": true
           },
           "stats" : {
             "pages_processed" : 0,
@@ -260,7 +262,8 @@ Which will yield the following response:
             "page_size" : 1000
           },
           "status" : {
-            "job_state" : "stopped"
+            "job_state" : "stopped",
+            "upgraded_doc_id": true
           },
           "stats" : {
             "pages_processed" : 0,
diff --git a/x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java b/x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java
index e534e4f41bbd6..366639d2482d5 100644
--- a/x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java
+++ b/x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java
@@ -58,8 +58,7 @@ protected ClientYamlTestClient initClientYamlTestClient(
             final List<HttpHost> hosts,
             final Version esVersion,
             final Version masterVersion) {
-        return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, masterVersion,
-                restClientBuilder -> configureClient(restClientBuilder, restClientSettings()));
+        return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, masterVersion, this::getClientBuilderWithSniffedHosts);
     }
 
     /**
diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index b498246391ded..a3b4bea9702f5 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -39,6 +39,10 @@ dependencies {
     // security deps
     shadow 'com.unboundid:unboundid-ldapsdk:3.2.0'
     shadow project(path: ':modules:transport-netty4', configuration: 'runtime')
+    shadow(project(path: ':plugins:transport-nio', configuration: 'runtime')) {
+        // TODO: core exclusion should not be necessary, since it is a transitive dep of all plugins
+        exclude group: "org.elasticsearch", module: "elasticsearch-core"
+    }
 
     testCompile 'org.elasticsearch:securemock:1.2'
     testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseAction.java
index a2c8d609be084..e8d5b3f3f5709 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseAction.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseResponse;
 
 public class DeleteLicenseAction extends Action<DeleteLicenseResponse> {
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseRequest.java
deleted file mode 100644
index 29558cf9e42bb..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseRequest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.support.master.AcknowledgedRequest;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-
-import java.io.IOException;
-
-
-public class DeleteLicenseRequest extends AcknowledgedRequest<DeleteLicenseRequest> {
-
-    public DeleteLicenseRequest() {
-    }
-
-    @Override
-    public ActionRequestValidationException validate() {
-        return null;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseRequestBuilder.java
index b554b0055376b..ad58cd3e1a5b9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseRequestBuilder.java
@@ -7,6 +7,8 @@
 
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseResponse;
 
 public class DeleteLicenseRequestBuilder extends AcknowledgedRequestBuilder<DeleteLicenseRequest, DeleteLicenseResponse,
         DeleteLicenseRequestBuilder> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseResponse.java
deleted file mode 100644
index 0dd092d6fe6ae..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseResponse.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-
-public class DeleteLicenseResponse extends AcknowledgedResponse {
-
-    DeleteLicenseResponse() {
-    }
-
-    DeleteLicenseResponse(boolean acknowledged) {
-        super(acknowledged);
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseRequest.java
deleted file mode 100644
index 914e18772af03..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseRequest.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
-import org.elasticsearch.common.io.stream.StreamInput;
-
-import java.io.IOException;
-
-
-public class GetLicenseRequest extends MasterNodeReadRequest<GetLicenseRequest> {
-
-    public GetLicenseRequest() {
-    }
-
-    public GetLicenseRequest(StreamInput in) throws IOException {
-        super(in);
-    }
-
-    @Override
-    public ActionRequestValidationException validate() {
-        return null;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseRequestBuilder.java
index 7e92a54bce2d6..ed72e67468731 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseRequestBuilder.java
@@ -7,6 +7,7 @@
 
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 
 public class GetLicenseRequestBuilder extends MasterNodeReadOperationRequestBuilder<GetLicenseRequest, GetLicenseResponse,
         GetLicenseRequestBuilder> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
index 505d2c55fa652..e087438924394 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.LicensesStatus;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -209,38 +210,44 @@ public void registerLicense(final PutLicenseRequest request, final ActionListene
                 }
             }
 
-            if (newLicense.isProductionLicense()
-                    && XPackSettings.SECURITY_ENABLED.get(settings)
+            if (XPackSettings.SECURITY_ENABLED.get(settings)) {
+                // TODO we should really validate that all nodes have xpack installed and are consistently configured but this
+                // should happen on a different level and not in this code
+                if (newLicense.isProductionLicense()
                     && XPackSettings.TRANSPORT_SSL_ENABLED.get(settings) == false
                     && isProductionMode(settings, clusterService.localNode())) {
-                // security is on but TLS is not configured we gonna fail the entire request and throw an exception
-                throw new IllegalStateException("Cannot install a [" + newLicense.operationMode() +
+                    // security is on but TLS is not configured we gonna fail the entire request and throw an exception
+                    throw new IllegalStateException("Cannot install a [" + newLicense.operationMode() +
                         "] license unless TLS is configured or security is disabled");
-                // TODO we should really validate that all nodes have xpack installed and are consistently configured but this
-                // should happen on a different level and not in this code
-            } else {
-                clusterService.submitStateUpdateTask("register license [" + newLicense.uid() + "]", new
-                        AckedClusterStateUpdateTask<PutLicenseResponse>(request, listener) {
-                            @Override
-                            protected PutLicenseResponse newResponse(boolean acknowledged) {
-                                return new PutLicenseResponse(acknowledged, LicensesStatus.VALID);
-                            }
+                } else if (XPackSettings.FIPS_MODE_ENABLED.get(settings)
+                    && newLicense.operationMode() != License.OperationMode.PLATINUM
+                    && newLicense.operationMode() != License.OperationMode.TRIAL) {
+                    throw new IllegalStateException("Cannot install a [" + newLicense.operationMode() +
+                        "] license unless FIPS mode is disabled");
+                }
+            }
+
+            clusterService.submitStateUpdateTask("register license [" + newLicense.uid() + "]", new
+                    AckedClusterStateUpdateTask<PutLicenseResponse>(request, listener) {
+                        @Override
+                        protected PutLicenseResponse newResponse(boolean acknowledged) {
+                            return new PutLicenseResponse(acknowledged, LicensesStatus.VALID);
+                        }
 
-                            @Override
-                            public ClusterState execute(ClusterState currentState) throws Exception {
-                                XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
-                                MetaData currentMetadata = currentState.metaData();
-                                LicensesMetaData licensesMetaData = currentMetadata.custom(LicensesMetaData.TYPE);
-                                Version trialVersion = null;
-                                if (licensesMetaData != null) {
-                                    trialVersion = licensesMetaData.getMostRecentTrialVersion();
-                                }
-                                MetaData.Builder mdBuilder = MetaData.builder(currentMetadata);
-                                mdBuilder.putCustom(LicensesMetaData.TYPE, new LicensesMetaData(newLicense, trialVersion));
-                                return ClusterState.builder(currentState).metaData(mdBuilder).build();
+                        @Override
+                        public ClusterState execute(ClusterState currentState) throws Exception {
+                            XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
+                            MetaData currentMetadata = currentState.metaData();
+                            LicensesMetaData licensesMetaData = currentMetadata.custom(LicensesMetaData.TYPE);
+                            Version trialVersion = null;
+                            if (licensesMetaData != null) {
+                                trialVersion = licensesMetaData.getMostRecentTrialVersion();
                             }
-                        });
-            }
+                            MetaData.Builder mdBuilder = MetaData.builder(currentMetadata);
+                            mdBuilder.putCustom(LicensesMetaData.TYPE, new LicensesMetaData(newLicense, trialVersion));
+                            return ClusterState.builder(currentState).metaData(mdBuilder).build();
+                        }
+                    });
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
index 14a059e9e014a..dfccd1d419054 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
@@ -7,6 +7,9 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseResponse;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 
 public class LicensingClient {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
index 7395f00649608..21d591d98ca90 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
index 31004823e4920..2d72bc3bed5f3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
index a1d57684a1737..4a6b5e0116f6f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
@@ -17,6 +17,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseResponse;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java
index ba6da84f19bb5..de55a66427769 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
index 8c4c5e2c7600b..1a80c6d873b53 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
@@ -87,6 +87,10 @@ private XPackSettings() {
     public static final Setting<Boolean> TOKEN_SERVICE_ENABLED_SETTING = Setting.boolSetting("xpack.security.authc.token.enabled",
         XPackSettings.HTTP_SSL_ENABLED::getRaw, Setting.Property.NodeScope);
 
+    /** Setting for enabling or disabling FIPS mode. Defaults to false */
+    public static final Setting<Boolean> FIPS_MODE_ENABLED =
+        Setting.boolSetting("xpack.security.fips_mode.enabled", false, Property.NodeScope);
+
     /** Setting for enabling or disabling sql. Defaults to true. */
     public static final Setting<Boolean> SQL_ENABLED = Setting.boolSetting("xpack.sql.enabled", true, Setting.Property.NodeScope);
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
index 371cdd8c0387a..5badd073c66c7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
@@ -64,7 +64,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
     private static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets");
     private static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window");
     private static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields");
-    private static final ParseField MULTIPLE_BUCKET_SPANS = new ParseField("multiple_bucket_spans");
     private static final ParseField USER_PER_PARTITION_NORMALIZATION = new ParseField("use_per_partition_normalization");
 
     public static final String ML_CATEGORY_FIELD = "mlcategory";
@@ -99,9 +98,6 @@ private static ConstructingObjectParser<AnalysisConfig.Builder, Void> createPars
         parser.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS);
         parser.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW);
         parser.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);
-        parser.declareStringArray((builder, values) -> builder.setMultipleBucketSpans(
-            values.stream().map(v -> TimeValue.parseTimeValue(v, MULTIPLE_BUCKET_SPANS.getPreferredName()))
-                .collect(Collectors.toList())), MULTIPLE_BUCKET_SPANS);
         parser.declareBoolean(Builder::setUsePerPartitionNormalization, USER_PER_PARTITION_NORMALIZATION);
 
         return parser;
@@ -121,13 +117,12 @@ private static ConstructingObjectParser<AnalysisConfig.Builder, Void> createPars
     private final Boolean overlappingBuckets;
     private final Long resultFinalizationWindow;
     private final Boolean multivariateByFields;
-    private final List<TimeValue> multipleBucketSpans;
     private final boolean usePerPartitionNormalization;
 
     private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List<String> categorizationFilters,
                            CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName,
                            List<Detector> detectors, List<String> influencers, Boolean overlappingBuckets, Long resultFinalizationWindow,
-                           Boolean multivariateByFields, List<TimeValue> multipleBucketSpans, boolean usePerPartitionNormalization) {
+                           Boolean multivariateByFields, boolean usePerPartitionNormalization) {
         this.detectors = detectors;
         this.bucketSpan = bucketSpan;
         this.latency = latency;
@@ -139,7 +134,6 @@ private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, Lis
         this.overlappingBuckets = overlappingBuckets;
         this.resultFinalizationWindow = resultFinalizationWindow;
         this.multivariateByFields = multivariateByFields;
-        this.multipleBucketSpans = multipleBucketSpans == null ? null : Collections.unmodifiableList(multipleBucketSpans);
         this.usePerPartitionNormalization = usePerPartitionNormalization;
     }
 
@@ -159,16 +153,18 @@ public AnalysisConfig(StreamInput in) throws IOException {
         overlappingBuckets = in.readOptionalBoolean();
         resultFinalizationWindow = in.readOptionalLong();
         multivariateByFields = in.readOptionalBoolean();
-        if (in.readBoolean()) {
-            final int arraySize = in.readVInt();
-            final List<TimeValue> spans = new ArrayList<>(arraySize);
-            for (int i = 0; i < arraySize; i++) {
-                spans.add(in.readTimeValue());
+
+        // BWC for removed multiple_bucket_spans
+        // TODO Remove in 7.0.0
+        if (in.getVersion().before(Version.V_6_5_0)) {
+            if (in.readBoolean()) {
+                final int arraySize = in.readVInt();
+                for (int i = 0; i < arraySize; i++) {
+                    in.readTimeValue();
+                }
             }
-            multipleBucketSpans = Collections.unmodifiableList(spans);
-        } else {
-            multipleBucketSpans = null;
         }
+
         usePerPartitionNormalization = in.readBoolean();
     }
 
@@ -192,15 +188,13 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalBoolean(overlappingBuckets);
         out.writeOptionalLong(resultFinalizationWindow);
         out.writeOptionalBoolean(multivariateByFields);
-        if (multipleBucketSpans != null) {
-            out.writeBoolean(true);
-            out.writeVInt(multipleBucketSpans.size());
-            for (TimeValue span : multipleBucketSpans) {
-                out.writeTimeValue(span);
-            }
-        } else {
+
+        // BWC for removed multiple_bucket_spans
+        // TODO Remove in 7.0.0
+        if (out.getVersion().before(Version.V_6_5_0)) {
             out.writeBoolean(false);
         }
+
         out.writeBoolean(usePerPartitionNormalization);
     }
 
@@ -305,10 +299,6 @@ public Boolean getMultivariateByFields() {
         return multivariateByFields;
     }
 
-    public List<TimeValue> getMultipleBucketSpans() {
-        return multipleBucketSpans;
-    }
-
     public boolean getUsePerPartitionNormalization() {
         return usePerPartitionNormalization;
     }
@@ -413,10 +403,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (multivariateByFields != null) {
             builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields);
         }
-        if (multipleBucketSpans != null) {
-            builder.field(MULTIPLE_BUCKET_SPANS.getPreferredName(),
-                    multipleBucketSpans.stream().map(TimeValue::getStringRep).collect(Collectors.toList()));
-        }
         if (usePerPartitionNormalization) {
             builder.field(USER_PER_PARTITION_NORMALIZATION.getPreferredName(), usePerPartitionNormalization);
         }
@@ -440,8 +426,7 @@ public boolean equals(Object o) {
                 Objects.equals(influencers, that.influencers) &&
                 Objects.equals(overlappingBuckets, that.overlappingBuckets) &&
                 Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) &&
-                Objects.equals(multivariateByFields, that.multivariateByFields) &&
-                Objects.equals(multipleBucketSpans, that.multipleBucketSpans);
+                Objects.equals(multivariateByFields, that.multivariateByFields);
     }
 
     @Override
@@ -449,7 +434,7 @@ public int hashCode() {
         return Objects.hash(
                 bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency,
                 summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow,
-                multivariateByFields, multipleBucketSpans, usePerPartitionNormalization
+                multivariateByFields, usePerPartitionNormalization
         );
     }
 
@@ -468,7 +453,6 @@ public static class Builder {
         private Boolean overlappingBuckets;
         private Long resultFinalizationWindow;
         private Boolean multivariateByFields;
-        private List<TimeValue> multipleBucketSpans;
         private boolean usePerPartitionNormalization = false;
 
         public Builder(List<Detector> detectors) {
@@ -488,8 +472,6 @@ public Builder(AnalysisConfig analysisConfig) {
             this.overlappingBuckets = analysisConfig.overlappingBuckets;
             this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow;
             this.multivariateByFields = analysisConfig.multivariateByFields;
-            this.multipleBucketSpans = analysisConfig.multipleBucketSpans == null ? null
-                    : new ArrayList<>(analysisConfig.multipleBucketSpans);
             this.usePerPartitionNormalization = analysisConfig.usePerPartitionNormalization;
         }
 
@@ -553,10 +535,6 @@ public void setMultivariateByFields(Boolean multivariateByFields) {
             this.multivariateByFields = multivariateByFields;
         }
 
-        public void setMultipleBucketSpans(List<TimeValue> multipleBucketSpans) {
-            this.multipleBucketSpans = multipleBucketSpans;
-        }
-
         public void setUsePerPartitionNormalization(boolean usePerPartitionNormalization) {
             this.usePerPartitionNormalization = usePerPartitionNormalization;
         }
@@ -588,7 +566,6 @@ public AnalysisConfig build() {
             verifyCategorizationAnalyzer();
             verifyCategorizationFilters();
             checkFieldIsNotNegativeIfSpecified(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow);
-            verifyMultipleBucketSpans();
 
             verifyNoMetricFunctionsWhenSummaryCountFieldNameIsSet();
 
@@ -603,7 +580,7 @@ public AnalysisConfig build() {
 
             return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig,
                     latency, summaryCountFieldName, detectors, influencers, overlappingBuckets,
-                    resultFinalizationWindow, multivariateByFields, multipleBucketSpans, usePerPartitionNormalization);
+                    resultFinalizationWindow, multivariateByFields, usePerPartitionNormalization);
         }
 
         private void verifyNoMetricFunctionsWhenSummaryCountFieldNameIsSet() {
@@ -727,19 +704,6 @@ private void verifyCategorizationFiltersAreValidRegex() {
             }
         }
 
-        private void verifyMultipleBucketSpans() {
-            if (multipleBucketSpans == null) {
-                return;
-            }
-
-            for (TimeValue span : multipleBucketSpans) {
-                if ((span.getSeconds() % bucketSpan.getSeconds() != 0L) || (span.compareTo(bucketSpan) <= 0)) {
-                    throw ExceptionsHelper.badRequestException(
-                            Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, span, bucketSpan));
-                }
-            }
-        }
-
         private static void checkDetectorsHavePartitionFields(List<Detector> detectors) {
             for (Detector detector : detectors) {
                 if (!Strings.isNullOrEmpty(detector.getPartitionFieldName())) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java
index 0c633c6ead22d..3663560b978f3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java
@@ -84,6 +84,10 @@ public boolean isEmpty() {
     public void validate(Set<String> validKeys) {
         Optional<String> invalidKey = scope.keySet().stream().filter(k -> !validKeys.contains(k)).findFirst();
         if (invalidKey.isPresent()) {
+            if (validKeys.isEmpty()) {
+                throw ExceptionsHelper.badRequestException(Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_SCOPE_NO_AVAILABLE_FIELDS,
+                        invalidKey.get()));
+            }
             throw ExceptionsHelper.badRequestException(Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_SCOPE_HAS_INVALID_FIELD,
                     invalidKey.get(), validKeys));
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
index 259d2d06a9c6e..289839b9342bc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
@@ -92,9 +92,11 @@ public final class Messages {
     public static final String JOB_CONFIG_CATEGORIZATION_ANALYZER_REQUIRES_CATEGORIZATION_FIELD_NAME =
             "categorization_analyzer requires setting categorization_field_name";
     public static final String JOB_CONFIG_DETECTION_RULE_NOT_SUPPORTED_BY_FUNCTION =
-            "Invalid detector rule: function {0} does not support rules with conditions";
+            "Invalid detector rule: function {0} only supports conditions that apply to time";
     public static final String JOB_CONFIG_DETECTION_RULE_REQUIRES_SCOPE_OR_CONDITION =
             "Invalid detector rule: at least scope or a condition is required";
+    public static final String JOB_CONFIG_DETECTION_RULE_SCOPE_NO_AVAILABLE_FIELDS =
+            "Invalid detector rule: scope field ''{0}'' is invalid; detector has no available fields for scoping";
     public static final String JOB_CONFIG_DETECTION_RULE_SCOPE_HAS_INVALID_FIELD =
             "Invalid detector rule: scope field ''{0}'' is invalid; select from {1}";
     public static final String JOB_CONFIG_FIELDNAME_INCOMPATIBLE_FUNCTION = "field_name cannot be used with function ''{0}''";
@@ -121,8 +123,6 @@ public final class Messages {
     public static final String JOB_CONFIG_INVALID_TIMEFORMAT = "Invalid Time format string ''{0}''";
     public static final String JOB_CONFIG_MISSING_ANALYSISCONFIG = "An analysis_config must be set";
     public static final String JOB_CONFIG_MISSING_DATA_DESCRIPTION = "A data_description must be set";
-    public static final String JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE =
-            "Multiple bucket_span ''{0}'' must be a multiple of the main bucket_span ''{1}''";
     public static final String JOB_CONFIG_ANALYSIS_FIELD_MUST_BE_SET =
             "Unless a count or temporal function is used one of field_name, by_field_name or over_field_name must be set";
     public static final String JOB_CONFIG_NO_DETECTORS = "No detectors configured";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java
index d5a5e7a07fadf..50f7931508585 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java
@@ -212,7 +212,7 @@ public static class JobWrapper implements Writeable, ToXContentObject {
                 (RollupJobStats) a[1], (RollupJobStatus)a[2]));
 
         static {
-            PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> RollupJobConfig.PARSER.apply(p,c).build(), CONFIG);
+            PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> RollupJobConfig.fromXContent(p, null), CONFIG);
             PARSER.declareObject(ConstructingObjectParser.constructorArg(), RollupJobStats.PARSER::apply, STATS);
             PARSER.declareObject(ConstructingObjectParser.constructorArg(), RollupJobStatus.PARSER::apply, STATUS);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java
index 9c3767d418856..962f4cceb6af5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java
@@ -52,9 +52,8 @@ public Request() {
 
         }
 
-        public static Request parseRequest(String id, XContentParser parser) {
-            RollupJobConfig.Builder config = RollupJobConfig.Builder.fromXContent(id, parser);
-            return new Request(config.build());
+        public static Request fromXContent(final XContentParser parser, final String id) throws IOException {
+            return new Request(RollupJobConfig.fromXContent(parser, id));
         }
 
         public RollupJobConfig getConfig() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java
index c8874ae459d2d..1b8eb736084a8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java
@@ -42,8 +42,8 @@ public RollupJobCaps(RollupJobConfig job) {
         jobID = job.getId();
         rollupIndex = job.getRollupIndex();
         indexPattern = job.getIndexPattern();
-        Map<String, Object> dateHistoAggCap = job.getGroupConfig().getDateHisto().toAggCap();
-        String dateField = job.getGroupConfig().getDateHisto().getField();
+        Map<String, Object> dateHistoAggCap = job.getGroupConfig().getDateHistogram().toAggCap();
+        String dateField = job.getGroupConfig().getDateHistogram().getField();
         RollupFieldCaps fieldCaps = fieldCapLookup.get(dateField);
         if (fieldCaps == null) {
             fieldCaps = new RollupFieldCaps();
@@ -51,9 +51,9 @@ public RollupJobCaps(RollupJobConfig job) {
         fieldCaps.addAgg(dateHistoAggCap);
         fieldCapLookup.put(dateField, fieldCaps);
 
-        if (job.getGroupConfig().getHisto() != null) {
-            Map<String, Object> histoAggCap = job.getGroupConfig().getHisto().toAggCap();
-            Arrays.stream(job.getGroupConfig().getHisto().getFields()).forEach(field -> {
+        if (job.getGroupConfig().getHistogram() != null) {
+            Map<String, Object> histoAggCap = job.getGroupConfig().getHistogram().toAggCap();
+            Arrays.stream(job.getGroupConfig().getHistogram().getFields()).forEach(field -> {
                 RollupFieldCaps caps = fieldCapLookup.get(field);
                 if (caps == null) {
                     caps = new RollupFieldCaps();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistoGroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfig.java
similarity index 53%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistoGroupConfig.java
rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfig.java
index 4a9fbde61d6be..77dfa1cbbb1c3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistoGroupConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfig.java
@@ -7,6 +7,7 @@
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -15,8 +16,8 @@
 import org.elasticsearch.common.rounding.DateTimeUnit;
 import org.elasticsearch.common.rounding.Rounding;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder;
@@ -33,6 +34,10 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+import static org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+
 /**
  * The configuration object for the histograms in the rollup config
  *
@@ -47,71 +52,100 @@
  *     ]
  * }
  */
-public class DateHistoGroupConfig implements Writeable, ToXContentFragment {
-    private static final String NAME = "date_histo_group_config";
-    public static final ObjectParser<DateHistoGroupConfig.Builder, Void> PARSER
-            = new ObjectParser<>(NAME, DateHistoGroupConfig.Builder::new);
-
-    private static final ParseField INTERVAL = new ParseField("interval");
-    private static final ParseField DELAY = new ParseField("delay");
-    private static final ParseField FIELD = new ParseField("field");
-    public static final ParseField TIME_ZONE = new ParseField("time_zone");
+public class DateHistogramGroupConfig implements Writeable, ToXContentObject {
+
+    static final String NAME = "date_histogram";
+    private static final String INTERVAL = "interval";
+    private static final String FIELD = "field";
+    public static final String TIME_ZONE = "time_zone";
+    private static final String DELAY = "delay";
+    private static final String DEFAULT_TIMEZONE = "UTC";
+    private static final ConstructingObjectParser<DateHistogramGroupConfig, Void> PARSER;
+    static {
+        PARSER = new ConstructingObjectParser<>(NAME, a ->
+            new DateHistogramGroupConfig((String) a[0], (DateHistogramInterval) a[1], (DateHistogramInterval) a[2], (String) a[3]));
+        PARSER.declareString(constructorArg(), new ParseField(FIELD));
+        PARSER.declareField(constructorArg(), p -> new DateHistogramInterval(p.text()), new ParseField(INTERVAL), ValueType.STRING);
+        PARSER.declareField(optionalConstructorArg(),  p -> new DateHistogramInterval(p.text()), new ParseField(DELAY), ValueType.STRING);
+        PARSER.declareString(optionalConstructorArg(), new ParseField(TIME_ZONE));
+    }
 
-    private final DateHistogramInterval interval;
     private final String field;
-    private final DateTimeZone timeZone;
+    private final DateHistogramInterval interval;
     private final DateHistogramInterval delay;
+    private final String timeZone;
 
-    static {
-        PARSER.declareField(DateHistoGroupConfig.Builder::setInterval,
-                p -> new DateHistogramInterval(p.text()), INTERVAL, ObjectParser.ValueType.STRING);
-        PARSER.declareString(DateHistoGroupConfig.Builder::setField, FIELD);
-        PARSER.declareField(DateHistoGroupConfig.Builder::setDelay,
-                p -> new DateHistogramInterval(p.text()), DELAY, ObjectParser.ValueType.LONG);
-        PARSER.declareField(DateHistoGroupConfig.Builder::setTimeZone, p -> {
-            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                return DateTimeZone.forID(p.text());
-            } else {
-                return DateTimeZone.forOffsetHours(p.intValue());
-            }
-        }, TIME_ZONE, ObjectParser.ValueType.LONG);
+    /**
+     * Create a new {@link DateHistogramGroupConfig} using the given field and interval parameters.
+     */
+    public DateHistogramGroupConfig(final String field, final DateHistogramInterval interval) {
+        this(field, interval, null, null);
     }
 
-    private DateHistoGroupConfig(DateHistogramInterval interval,
-                                 String field,
-                                 DateHistogramInterval delay,
-                                 DateTimeZone timeZone) {
+    /**
+     * Create a new {@link DateHistogramGroupConfig} using the given configuration parameters.
+     * <p>
+     *     The {@code field} and {@code interval} are required to compute the date histogram for the rolled up documents.
+     *     The {@code delay} is optional and can be set to {@code null}. It defines how long to wait before rolling up new documents.
+     *     The {@code timeZone} is optional and can be set to {@code null}. When configured, the time zone value  is resolved using
+     *     ({@link DateTimeZone#forID(String)} and must match a time zone identifier provided by the Joda Time library.
+     * </p>
+     * @param field the name of the date field to use for the date histogram (required)
+     * @param interval the interval to use for the date histogram (required)
+     * @param delay the time delay (optional)
+     * @param timeZone the id of time zone to use to calculate the date histogram (optional). When {@code null}, the UTC timezone is used.
+     */
+    public DateHistogramGroupConfig(final String field,
+                                    final DateHistogramInterval interval,
+                                    final @Nullable DateHistogramInterval delay,
+                                    final @Nullable String timeZone) {
+        if (field == null || field.isEmpty()) {
+            throw new IllegalArgumentException("Field must be a non-null, non-empty string");
+        }
+        if (interval == null) {
+            throw new IllegalArgumentException("Interval must be non-null");
+        }
+
         this.interval = interval;
         this.field = field;
         this.delay = delay;
-        this.timeZone = Objects.requireNonNull(timeZone);
+        this.timeZone = (timeZone != null && timeZone.isEmpty() == false) ? timeZone : DEFAULT_TIMEZONE;
+
+        // validate interval
+        createRounding(this.interval.toString(), this.timeZone);
+        if (delay != null) {
+            // and delay
+            TimeValue.parseTimeValue(this.delay.toString(), DELAY);
+        }
     }
 
-    DateHistoGroupConfig(StreamInput in) throws IOException {
+    DateHistogramGroupConfig(final StreamInput in) throws IOException {
         interval = new DateHistogramInterval(in);
         field = in.readString();
         delay = in.readOptionalWriteable(DateHistogramInterval::new);
-        timeZone = in.readTimeZone();
+        timeZone = in.readString();
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    public void writeTo(final StreamOutput out) throws IOException {
         interval.writeTo(out);
         out.writeString(field);
         out.writeOptionalWriteable(delay);
-        out.writeTimeZone(timeZone);
+        out.writeString(timeZone);
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(INTERVAL.getPreferredName(), interval.toString());
-        builder.field(FIELD.getPreferredName(), field);
-        if (delay != null) {
-            builder.field(DELAY.getPreferredName(), delay.toString());
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.field(INTERVAL, interval.toString());
+            builder.field(FIELD, field);
+            if (delay != null) {
+                builder.field(DELAY, delay.toString());
+            }
+            builder.field(TIME_ZONE, timeZone);
         }
-        builder.field(TIME_ZONE.getPreferredName(), timeZone.toString());
-
-        return builder;
+        return builder.endObject();
     }
 
     /**
@@ -138,7 +172,7 @@ public DateHistogramInterval getDelay() {
     /**
      * Get the timezone to apply
      */
-    public DateTimeZone getTimeZone() {
+    public String getTimeZone() {
         return timeZone;
     }
 
@@ -146,9 +180,9 @@ public DateTimeZone getTimeZone() {
      * Create the rounding for this date histogram
      */
     public Rounding createRounding() {
-        return createRounding(interval.toString(), timeZone, "");
+        return createRounding(interval.toString(), timeZone);
     }
-    ;
+
     /**
      * This returns a set of aggregation builders which represent the configured
      * set of date histograms.  Used by the rollup indexer to iterate over historical data
@@ -158,7 +192,7 @@ public List<CompositeValuesSourceBuilder<?>> toBuilders() {
                 new DateHistogramValuesSourceBuilder(RollupField.formatIndexerAggName(field, DateHistogramAggregationBuilder.NAME));
         vsBuilder.dateHistogramInterval(interval);
         vsBuilder.field(field);
-        vsBuilder.timeZone(timeZone);
+        vsBuilder.timeZone(toDateTimeZone(timeZone));
         return Collections.singletonList(vsBuilder);
     }
 
@@ -168,11 +202,11 @@ public List<CompositeValuesSourceBuilder<?>> toBuilders() {
     public Map<String, Object> toAggCap() {
         Map<String, Object> map = new HashMap<>(3);
         map.put("agg", DateHistogramAggregationBuilder.NAME);
-        map.put(INTERVAL.getPreferredName(), interval.toString());
+        map.put(INTERVAL, interval.toString());
         if (delay != null) {
-            map.put(DELAY.getPreferredName(), delay.toString());
+            map.put(DELAY, delay.toString());
         }
-        map.put(TIME_ZONE.getPreferredName(), timeZone.toString());
+        map.put(TIME_ZONE, timeZone);
 
         return map;
     }
@@ -204,21 +238,18 @@ public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCa
     }
 
     @Override
-    public boolean equals(Object other) {
+    public boolean equals(final Object other) {
         if (this == other) {
             return true;
         }
-
         if (other == null || getClass() != other.getClass()) {
             return false;
         }
-
-        DateHistoGroupConfig that = (DateHistoGroupConfig) other;
-
-        return Objects.equals(this.interval, that.interval)
-                && Objects.equals(this.field, that.field)
-                && Objects.equals(this.delay, that.delay)
-                && Objects.equals(this.timeZone, that.timeZone);
+        final DateHistogramGroupConfig that = (DateHistogramGroupConfig) other;
+        return Objects.equals(interval, that.interval)
+            && Objects.equals(field, that.field)
+            && Objects.equals(delay, that.delay)
+            && Objects.equals(timeZone, that.timeZone);
     }
 
     @Override
@@ -231,77 +262,28 @@ public String toString() {
         return Strings.toString(this, true, true);
     }
 
-    private static Rounding createRounding(String expr, DateTimeZone timeZone, String settingName) {
+    public static DateHistogramGroupConfig fromXContent(final XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    private static Rounding createRounding(final String expr, final String timeZone) {
         DateTimeUnit timeUnit = DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expr);
         final Rounding.Builder rounding;
         if (timeUnit != null) {
             rounding = new Rounding.Builder(timeUnit);
         } else {
-            rounding = new Rounding.Builder(TimeValue.parseTimeValue(expr, settingName));
+            rounding = new Rounding.Builder(TimeValue.parseTimeValue(expr, "createRounding"));
         }
-        rounding.timeZone(timeZone);
+        rounding.timeZone(toDateTimeZone(timeZone));
         return rounding.build();
     }
 
-    public static class Builder {
-        private DateHistogramInterval interval;
-        private String field;
-        private DateHistogramInterval delay;
-        private DateTimeZone timeZone;
-
-        public DateHistogramInterval getInterval() {
-            return interval;
-        }
-
-        public DateHistoGroupConfig.Builder setInterval(DateHistogramInterval interval) {
-            this.interval = interval;
-            return this;
-        }
-
-        public String getField() {
-            return field;
-        }
-
-        public DateHistoGroupConfig.Builder setField(String field) {
-            this.field = field;
-            return this;
-        }
-
-        public DateTimeZone getTimeZone() {
-            return timeZone;
-        }
-
-        public DateHistoGroupConfig.Builder setTimeZone(DateTimeZone timeZone) {
-            this.timeZone = timeZone;
-            return this;
-        }
-
-        public DateHistogramInterval getDelay() {
-            return delay;
-        }
-
-        public DateHistoGroupConfig.Builder setDelay(DateHistogramInterval delay) {
-            this.delay = delay;
-            return this;
-        }
-
-        public DateHistoGroupConfig build() {
-            if (field == null || field.isEmpty()) {
-                throw new IllegalArgumentException("Parameter [" + FIELD.getPreferredName() + "] is mandatory.");
-            }
-            if (timeZone == null) {
-                timeZone = DateTimeZone.UTC;
-            }
-            if (interval == null) {
-                throw new IllegalArgumentException("Parameter [" + INTERVAL.getPreferredName() + "] is mandatory.");
-            }
-            // validate interval
-            createRounding(interval.toString(), timeZone, INTERVAL.getPreferredName());
-            if (delay != null) {
-                // and delay
-                TimeValue.parseTimeValue(delay.toString(), INTERVAL.getPreferredName());
-            }
-            return new DateHistoGroupConfig(interval, field, delay, timeZone);
+    private static DateTimeZone toDateTimeZone(final String timezone) {
+        try {
+            return DateTimeZone.forOffsetHours(Integer.parseInt(timezone));
+        } catch (NumberFormatException e) {
+            return DateTimeZone.forID(timezone);
         }
     }
+
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java
index dcb393b088618..f7685f4e6143b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java
@@ -13,16 +13,22 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
+import static java.util.Arrays.asList;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
 /**
  * The configuration object for the groups section in the rollup config.
  * Basically just a wrapper for histo/date histo/terms objects
@@ -36,118 +42,131 @@
  * }
  */
 public class GroupConfig implements Writeable, ToXContentObject {
-    private static final String NAME = "grouping_config";
-    private static final ParseField DATE_HISTO = new ParseField("date_histogram");
-    private static final ParseField HISTO = new ParseField("histogram");
-    private static final ParseField TERMS = new ParseField("terms");
 
-    private final DateHistoGroupConfig dateHisto;
-    private final HistoGroupConfig histo;
-    private final TermsGroupConfig terms;
+    public static final String NAME = "groups";
+    private static final ConstructingObjectParser<GroupConfig, Void> PARSER;
+    static {
+        PARSER = new ConstructingObjectParser<>(NAME, args ->
+            new GroupConfig((DateHistogramGroupConfig) args[0], (HistogramGroupConfig) args[1], (TermsGroupConfig) args[2]));
+        PARSER.declareObject(constructorArg(),
+            (p, c) -> DateHistogramGroupConfig.fromXContent(p), new ParseField(DateHistogramGroupConfig.NAME));
+        PARSER.declareObject(optionalConstructorArg(),
+            (p, c) -> HistogramGroupConfig.fromXContent(p), new ParseField(HistogramGroupConfig.NAME));
+        PARSER.declareObject(optionalConstructorArg(),
+            (p, c) -> TermsGroupConfig.fromXContent(p), new ParseField(TermsGroupConfig.NAME));
+    }
 
-    public static final ObjectParser<GroupConfig.Builder, Void> PARSER = new ObjectParser<>(NAME, GroupConfig.Builder::new);
+    private final DateHistogramGroupConfig dateHistogram;
+    private final @Nullable HistogramGroupConfig histogram;
+    private final @Nullable TermsGroupConfig terms;
 
-    static {
-        PARSER.declareObject(GroupConfig.Builder::setDateHisto, (p,c) -> DateHistoGroupConfig.PARSER.apply(p,c).build(), DATE_HISTO);
-        PARSER.declareObject(GroupConfig.Builder::setHisto, (p,c) -> HistoGroupConfig.PARSER.apply(p,c).build(), HISTO);
-        PARSER.declareObject(GroupConfig.Builder::setTerms, (p,c) -> TermsGroupConfig.PARSER.apply(p,c).build(), TERMS);
+    public GroupConfig(final DateHistogramGroupConfig dateHistogram) {
+        this(dateHistogram, null, null);
     }
 
-    private GroupConfig(DateHistoGroupConfig dateHisto, @Nullable HistoGroupConfig histo, @Nullable TermsGroupConfig terms) {
-        this.dateHisto = Objects.requireNonNull(dateHisto, "A date_histogram group is mandatory");
-        this.histo = histo;
+    public GroupConfig(final DateHistogramGroupConfig dateHistogram,
+                       final @Nullable HistogramGroupConfig histogram,
+                       final @Nullable TermsGroupConfig terms) {
+        if (dateHistogram == null) {
+            throw new IllegalArgumentException("Date histogram must not be null");
+        }
+        this.dateHistogram = dateHistogram;
+        this.histogram = histogram;
         this.terms = terms;
     }
 
-    GroupConfig(StreamInput in) throws IOException {
-        dateHisto = new DateHistoGroupConfig(in);
-        histo = in.readOptionalWriteable(HistoGroupConfig::new);
+    GroupConfig(final StreamInput in) throws IOException {
+        dateHistogram = new DateHistogramGroupConfig(in);
+        histogram = in.readOptionalWriteable(HistogramGroupConfig::new);
         terms = in.readOptionalWriteable(TermsGroupConfig::new);
     }
 
-    public DateHistoGroupConfig getDateHisto() {
-        return dateHisto;
+    /**
+     * @return the configuration of the date histogram
+     */
+    public DateHistogramGroupConfig getDateHistogram() {
+        return dateHistogram;
     }
 
-    public HistoGroupConfig getHisto() {
-        return histo;
+    /**
+     * @return the configuration of the histogram
+     */
+    @Nullable
+    public HistogramGroupConfig getHistogram() {
+        return histogram;
     }
 
+    /**
+     * @return the configuration of the terms
+     */
+    @Nullable
     public TermsGroupConfig getTerms() {
         return terms;
     }
 
     public Set<String> getAllFields() {
         Set<String> fields = new HashSet<>();
-        fields.add(dateHisto.getField());
-        if (histo != null) {
-            fields.addAll(histo.getAllFields());
+        fields.add(dateHistogram.getField());
+        if (histogram != null) {
+            fields.addAll(asList(histogram.getFields()));
         }
         if (terms != null) {
-            fields.addAll(terms.getAllFields());
+            fields.addAll(asList(terms.getFields()));
         }
-        return fields;
+        return Collections.unmodifiableSet(fields);
     }
 
-    public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCapsResponse,
-                                                             ActionRequestValidationException validationException) {
-        dateHisto.validateMappings(fieldCapsResponse, validationException);
-        if (histo != null) {
-            histo.validateMappings(fieldCapsResponse, validationException);
+    public void validateMappings(final Map<String, Map<String, FieldCapabilities>> fieldCapsResponse,
+                                 final ActionRequestValidationException validationException) {
+        dateHistogram.validateMappings(fieldCapsResponse, validationException);
+        if (histogram != null) {
+            histogram.validateMappings(fieldCapsResponse, validationException);
         }
         if (terms != null) {
             terms.validateMappings(fieldCapsResponse, validationException);
         }
     }
 
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.startObject(DATE_HISTO.getPreferredName());
-        dateHisto.toXContent(builder, params);
-        builder.endObject();
-        if (histo != null) {
-            builder.startObject(HISTO.getPreferredName());
-            histo.toXContent(builder, params);
-            builder.endObject();
-        }
-        if (terms != null) {
-            builder.startObject(TERMS.getPreferredName());
-            terms.toXContent(builder, params);
-            builder.endObject();
+        {
+            builder.field(DateHistogramGroupConfig.NAME, dateHistogram);
+            if (histogram != null) {
+                builder.field(HistogramGroupConfig.NAME, histogram);
+            }
+            if (terms != null) {
+                builder.field(TermsGroupConfig.NAME, terms);
+            }
         }
-        builder.endObject();
-        return builder;
+        return builder.endObject();
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        dateHisto.writeTo(out);
-        out.writeOptionalWriteable(histo);
+    public void writeTo(final StreamOutput out) throws IOException {
+        dateHistogram.writeTo(out);
+        out.writeOptionalWriteable(histogram);
         out.writeOptionalWriteable(terms);
     }
 
     @Override
-    public boolean equals(Object other) {
+    public boolean equals(final Object other) {
         if (this == other) {
             return true;
         }
-
         if (other == null || getClass() != other.getClass()) {
             return false;
         }
 
-        GroupConfig that = (GroupConfig) other;
-
-        return Objects.equals(this.dateHisto, that.dateHisto)
-                && Objects.equals(this.histo, that.histo)
-                && Objects.equals(this.terms, that.terms);
+        final GroupConfig that = (GroupConfig) other;
+        return Objects.equals(dateHistogram, that.dateHistogram)
+            && Objects.equals(histogram, that.histogram)
+            && Objects.equals(terms, that.terms);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(dateHisto, histo, terms);
+        return Objects.hash(dateHistogram, histogram, terms);
     }
 
     @Override
@@ -155,43 +174,7 @@ public String toString() {
         return Strings.toString(this, true, true);
     }
 
-    public static class Builder {
-        private DateHistoGroupConfig dateHisto;
-        private HistoGroupConfig histo;
-        private TermsGroupConfig terms;
-
-        public DateHistoGroupConfig getDateHisto() {
-            return dateHisto;
-        }
-
-        public GroupConfig.Builder setDateHisto(DateHistoGroupConfig dateHisto) {
-            this.dateHisto = dateHisto;
-            return this;
-        }
-
-        public HistoGroupConfig getHisto() {
-            return histo;
-        }
-
-        public GroupConfig.Builder setHisto(HistoGroupConfig histo) {
-            this.histo = histo;
-            return this;
-        }
-
-        public TermsGroupConfig getTerms() {
-            return terms;
-        }
-
-        public GroupConfig.Builder setTerms(TermsGroupConfig terms) {
-            this.terms = terms;
-            return this;
-        }
-
-        public GroupConfig build() {
-            if (dateHisto == null) {
-                throw new IllegalArgumentException("A date_histogram group is mandatory");
-            }
-            return new GroupConfig(dateHisto, histo, terms);
-        }
+    public static GroupConfig fromXContent(final XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistogramGroupConfig.java
similarity index 67%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfig.java
rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistogramGroupConfig.java
index 87de9e165345e..0480050bf52f0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistogramGroupConfig.java
@@ -12,9 +12,10 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder;
 import org.elasticsearch.search.aggregations.bucket.composite.HistogramValuesSourceBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
@@ -27,9 +28,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
 /**
  * The configuration object for the histograms in the rollup config
  *
@@ -42,28 +44,36 @@
  *     ]
  * }
  */
-public class HistoGroupConfig implements Writeable, ToXContentFragment {
-    private static final String NAME = "histo_group_config";
-    public static final ObjectParser<HistoGroupConfig.Builder, Void> PARSER
-            = new ObjectParser<>(NAME, HistoGroupConfig.Builder::new);
+public class HistogramGroupConfig implements Writeable, ToXContentObject {
 
-    private static final ParseField INTERVAL = new ParseField("interval");
-    private static final ParseField FIELDS = new ParseField("fields");
+    static final String NAME = "histogram";
+    private static final String INTERVAL = "interval";
+    private static final String FIELDS = "fields";
+    private static final ConstructingObjectParser<HistogramGroupConfig, Void> PARSER;
+    static {
+        PARSER = new ConstructingObjectParser<>(NAME, args -> {
+            @SuppressWarnings("unchecked") List<String> fields = (List<String>) args[1];
+            return new HistogramGroupConfig((long) args[0], fields != null ? fields.toArray(new String[fields.size()]) : null);
+        });
+        PARSER.declareLong(constructorArg(), new ParseField(INTERVAL));
+        PARSER.declareStringArray(constructorArg(), new ParseField(FIELDS));
+    }
 
     private final long interval;
     private final String[] fields;
 
-    static {
-        PARSER.declareLong(HistoGroupConfig.Builder::setInterval, INTERVAL);
-        PARSER.declareStringArray(HistoGroupConfig.Builder::setFields, FIELDS);
-    }
-
-    private HistoGroupConfig(long interval, String[] fields) {
+    public HistogramGroupConfig(final long interval, final String... fields) {
+        if (interval <= 0) {
+            throw new IllegalArgumentException("Interval must be a positive long");
+        }
+        if (fields == null || fields.length == 0) {
+            throw new IllegalArgumentException("Fields must have at least one value");
+        }
         this.interval = interval;
         this.fields = fields;
     }
 
-    HistoGroupConfig(StreamInput in) throws IOException {
+    HistogramGroupConfig(final StreamInput in) throws IOException {
         interval = in.readVLong();
         fields = in.readStringArray();
     }
@@ -101,7 +111,7 @@ public List<CompositeValuesSourceBuilder<?>> toBuilders() {
     public Map<String, Object> toAggCap() {
         Map<String, Object> map = new HashMap<>(2);
         map.put("agg", HistogramAggregationBuilder.NAME);
-        map.put(INTERVAL.getPreferredName(), interval);
+        map.put(INTERVAL, interval);
         return map;
     }
 
@@ -109,10 +119,6 @@ public Map<String, Object> getMetadata() {
         return Collections.singletonMap(RollupField.formatMetaField(RollupField.INTERVAL), interval);
     }
 
-    public Set<String> getAllFields() {
-        return Arrays.stream(fields).collect(Collectors.toSet());
-    }
-
     public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCapsResponse,
                                  ActionRequestValidationException validationException) {
 
@@ -138,9 +144,13 @@ public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCa
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(INTERVAL.getPreferredName(), interval);
-        builder.field(FIELDS.getPreferredName(), fields);
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.field(INTERVAL, interval);
+            builder.field(FIELDS, fields);
+        }
+        builder.endObject();
         return builder;
     }
 
@@ -151,19 +161,15 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public boolean equals(Object other) {
+    public boolean equals(final Object other) {
         if (this == other) {
             return true;
         }
-
         if (other == null || getClass() != other.getClass()) {
             return false;
         }
-
-        HistoGroupConfig that = (HistoGroupConfig) other;
-
-        return Objects.equals(this.interval, that.interval)
-                && Arrays.equals(this.fields, that.fields);
+        final HistogramGroupConfig that = (HistogramGroupConfig) other;
+        return Objects.equals(interval, that.interval) && Arrays.equals(fields, that.fields);
     }
 
     @Override
@@ -176,36 +182,7 @@ public String toString() {
         return Strings.toString(this, true, true);
     }
 
-    public static class Builder {
-        private long interval = 0;
-        private List<String> fields;
-
-        public long getInterval() {
-            return interval;
-        }
-
-        public HistoGroupConfig.Builder setInterval(long interval) {
-            this.interval = interval;
-            return this;
-        }
-
-        public List<String> getFields() {
-            return fields;
-        }
-
-        public HistoGroupConfig.Builder setFields(List<String> fields) {
-            this.fields = fields;
-            return this;
-        }
-
-        public HistoGroupConfig build() {
-            if (interval <= 0) {
-                throw new IllegalArgumentException("Parameter [" + INTERVAL.getPreferredName() + "] must be a positive long.");
-            }
-            if (fields == null || fields.isEmpty()) {
-                throw new IllegalArgumentException("Parameter [" + FIELDS + "] must have at least one value.");
-            }
-            return new HistoGroupConfig(interval, fields.toArray(new String[0]));
-        }
+    public static HistogramGroupConfig fromXContent(final XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java
index 006d8c35c324d..cc673c4ed0d35 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java
@@ -12,9 +12,10 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
@@ -32,6 +33,8 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
 /**
  * The configuration object for the metrics portion of a rollup job config
  *
@@ -48,14 +51,7 @@
  *     ]
  * }
  */
-public class MetricConfig implements Writeable, ToXContentFragment {
-    private static final String NAME = "metric_config";
-
-    private String field;
-    private List<String> metrics;
-
-    private static final ParseField FIELD = new ParseField("field");
-    private static final ParseField METRICS = new ParseField("metrics");
+public class MetricConfig implements Writeable, ToXContentObject {
 
     // TODO: replace these with an enum
     private static final ParseField MIN = new ParseField("min");
@@ -64,27 +60,54 @@ public class MetricConfig implements Writeable, ToXContentFragment {
     private static final ParseField AVG = new ParseField("avg");
     private static final ParseField VALUE_COUNT = new ParseField("value_count");
 
-    public static final ObjectParser<MetricConfig.Builder, Void> PARSER = new ObjectParser<>(NAME, MetricConfig.Builder::new);
-
+    static final String NAME = "metrics";
+    private static final String FIELD = "field";
+    private static final String METRICS = "metrics";
+    private static final ConstructingObjectParser<MetricConfig, Void> PARSER;
     static {
-        PARSER.declareString(MetricConfig.Builder::setField, FIELD);
-        PARSER.declareStringArray(MetricConfig.Builder::setMetrics, METRICS);
+        PARSER = new ConstructingObjectParser<>(NAME, args -> {
+            @SuppressWarnings("unchecked") List<String> metrics = (List<String>) args[1];
+            return new MetricConfig((String) args[0], metrics);
+        });
+        PARSER.declareString(constructorArg(), new ParseField(FIELD));
+        PARSER.declareStringArray(constructorArg(), new ParseField(METRICS));
     }
 
-    MetricConfig(String name, List<String> metrics) {
-        this.field = name;
+    private final String field;
+    private final List<String> metrics;
+
+    public MetricConfig(final String field, final List<String> metrics) {
+        if (field == null || field.isEmpty()) {
+            throw new IllegalArgumentException("Field must be a non-null, non-empty string");
+        }
+        if (metrics == null || metrics.isEmpty()) {
+            throw new IllegalArgumentException("Metrics must be a non-null, non-empty array of strings");
+        }
+        metrics.forEach(m -> {
+            if (RollupField.SUPPORTED_METRICS.contains(m) == false) {
+                throw new IllegalArgumentException("Unsupported metric [" + m + "]. " +
+                    "Supported metrics include: " + RollupField.SUPPORTED_METRICS);
+            }
+        });
+        this.field = field;
         this.metrics = metrics;
     }
 
-    MetricConfig(StreamInput in) throws IOException {
+    MetricConfig(final StreamInput in) throws IOException {
         field = in.readString();
         metrics = in.readList(StreamInput::readString);
     }
 
+    /**
+     * @return the name of the field used in the metric configuration. Never {@code null}.
+     */
     public String getField() {
         return field;
     }
 
+    /**
+     * @return the names of the metrics used in the metric configuration. Never {@code null}.
+     */
     public List<String> getMetrics() {
         return metrics;
     }
@@ -159,10 +182,13 @@ public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCa
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(FIELD.getPreferredName(), field);
-        builder.field(METRICS.getPreferredName(), metrics);
-        return builder;
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.field(FIELD, field);
+            builder.field(METRICS, metrics);
+        }
+        return builder.endObject();
     }
 
     @Override
@@ -172,19 +198,16 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public boolean equals(Object other) {
+    public boolean equals(final Object other) {
         if (this == other) {
             return true;
         }
-
         if (other == null || getClass() != other.getClass()) {
             return false;
         }
 
-        MetricConfig that = (MetricConfig) other;
-
-        return Objects.equals(this.field, that.field)
-                && Objects.equals(this.metrics, that.metrics);
+        final MetricConfig that = (MetricConfig) other;
+        return Objects.equals(field, that.field) && Objects.equals(metrics, that.metrics);
     }
 
     @Override
@@ -197,52 +220,7 @@ public String toString() {
         return Strings.toString(this, true, true);
     }
 
-
-    public static class Builder {
-        private String field;
-        private List<String> metrics;
-
-        public Builder() {
-        }
-
-        public Builder(MetricConfig config) {
-            this.field = config.getField();
-            this.metrics = config.getMetrics();
-        }
-
-        public String getField() {
-            return field;
-        }
-
-        public MetricConfig.Builder setField(String field) {
-            this.field = field;
-            return this;
-        }
-
-        public List<String> getMetrics() {
-            return metrics;
-        }
-
-        public MetricConfig.Builder setMetrics(List<String> metrics) {
-            this.metrics = metrics;
-            return this;
-        }
-
-        public MetricConfig build() {
-            if (Strings.isNullOrEmpty(field) == true) {
-                throw new IllegalArgumentException("Parameter [" + FIELD.getPreferredName() + "] must be a non-null, non-empty string.");
-            }
-            if (metrics == null || metrics.isEmpty()) {
-                throw new IllegalArgumentException("Parameter [" + METRICS.getPreferredName()
-                        + "] must be a non-null, non-empty array of strings.");
-            }
-            metrics.forEach(m -> {
-                if (RollupField.SUPPORTED_METRICS.contains(m) == false) {
-                    throw new IllegalArgumentException("Unsupported metric [" + m + "].  " +
-                            "Supported metrics include: " + RollupField.SUPPORTED_METRICS);
-                }
-            });
-            return new MetricConfig(field, metrics);
-        }
+    public static MetricConfig fromXContent(final XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java
index 7afcdb71b11cc..94306966a34da 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java
@@ -41,7 +41,7 @@ public class RollupJob extends AbstractDiffable<RollupJob> implements XPackPlugi
             = new ConstructingObjectParser<>(NAME, a -> new RollupJob((RollupJobConfig) a[0], (Map<String, String>) a[1]));
 
     static {
-        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> RollupJobConfig.PARSER.apply(p,c).build(), CONFIG);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> RollupJobConfig.fromXContent(p, null), CONFIG);
         PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> p.mapStrings(), HEADERS);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java
index 422ecdd5fd9fb..27461c62b6724 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java
@@ -7,20 +7,19 @@
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.rollup.RollupField;
 import org.elasticsearch.xpack.core.scheduler.Cron;
 
 import java.io.IOException;
@@ -30,61 +29,112 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.stream.Collectors;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 /**
  * This class holds the configuration details of a rollup job, such as the groupings, metrics, what
  * index to rollup and where to roll them to.
  */
 public class RollupJobConfig implements NamedWriteable, ToXContentObject {
-    private static final String NAME = "xpack/rollup/jobconfig";
-
-    public static final ParseField TIMEOUT = new ParseField("timeout");
-    public static final ParseField CURRENT = new ParseField("current");
-    public static final ParseField CRON = new ParseField("cron");
-    public static final ParseField PAGE_SIZE = new ParseField("page_size");
-
-    private static final ParseField INDEX_PATTERN = new ParseField("index_pattern");
-    private static final ParseField ROLLUP_INDEX = new ParseField("rollup_index");
-    private static final ParseField GROUPS = new ParseField("groups");
-    private static final ParseField METRICS = new ParseField("metrics");
-
-    private String id;
-    private String indexPattern;
-    private String rollupIndex;
-    private GroupConfig groupConfig;
-    private List<MetricConfig> metricsConfig = Collections.emptyList();
-    private TimeValue timeout = TimeValue.timeValueSeconds(20);
-    private String cron;
-    private int pageSize;
-
-    public static final ObjectParser<RollupJobConfig.Builder, Void> PARSER = new ObjectParser<>(NAME, false, RollupJobConfig.Builder::new);
 
+    private static final String NAME = "xpack/rollup/jobconfig";
+    private static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(20);
+    private static final String ID = "id";
+    private static final String TIMEOUT = "timeout";
+    private static final String CRON = "cron";
+    private static final String PAGE_SIZE = "page_size";
+    private static final String INDEX_PATTERN = "index_pattern";
+    private static final String ROLLUP_INDEX = "rollup_index";
+
+    private final String id;
+    private final String indexPattern;
+    private final String rollupIndex;
+    private final GroupConfig groupConfig;
+    private final List<MetricConfig> metricsConfig;
+    private final TimeValue timeout;
+    private final String cron;
+    private final int pageSize;
+
+    private static final ConstructingObjectParser<RollupJobConfig, String> PARSER;
     static {
-        PARSER.declareString(RollupJobConfig.Builder::setId, RollupField.ID);
-        PARSER.declareObject(RollupJobConfig.Builder::setGroupConfig, (p, c) -> GroupConfig.PARSER.apply(p,c).build(), GROUPS);
-        PARSER.declareObjectArray(RollupJobConfig.Builder::setMetricsConfig, (p, c) -> MetricConfig.PARSER.apply(p, c).build(), METRICS);
-        PARSER.declareString((params, val) ->
-                params.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT);
-        PARSER.declareString(RollupJobConfig.Builder::setIndexPattern, INDEX_PATTERN);
-        PARSER.declareString(RollupJobConfig.Builder::setRollupIndex, ROLLUP_INDEX);
-        PARSER.declareString(RollupJobConfig.Builder::setCron, CRON);
-        PARSER.declareInt(RollupJobConfig.Builder::setPageSize, PAGE_SIZE);
+        PARSER = new ConstructingObjectParser<>(NAME, false, (args, optionalId) -> {
+            String id = args[0] != null ? (String) args[0] : optionalId;
+            String indexPattern = (String) args[1];
+            String rollupIndex = (String) args[2];
+            GroupConfig groupConfig = (GroupConfig) args[3];
+            @SuppressWarnings("unchecked")
+            List<MetricConfig> metricsConfig = (List<MetricConfig>) args[4];
+            TimeValue timeout = (TimeValue) args[5];
+            String cron = (String) args[6];
+            int pageSize = (int) args[7];
+            return new RollupJobConfig(id, indexPattern, rollupIndex, cron, pageSize, groupConfig, metricsConfig, timeout);
+        });
+        PARSER.declareString(optionalConstructorArg(), new ParseField(ID));
+        PARSER.declareString(constructorArg(), new ParseField(INDEX_PATTERN));
+        PARSER.declareString(constructorArg(), new ParseField(ROLLUP_INDEX));
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> GroupConfig.fromXContent(p), new ParseField(GroupConfig.NAME));
+        PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> MetricConfig.fromXContent(p), new ParseField(MetricConfig.NAME));
+        PARSER.declareField(optionalConstructorArg(), (p, c) -> TimeValue.parseTimeValue(p.textOrNull(), TIMEOUT),
+            new ParseField(TIMEOUT), ObjectParser.ValueType.STRING_OR_NULL);
+        PARSER.declareString(constructorArg(), new ParseField(CRON));
+        PARSER.declareInt(constructorArg(), new ParseField(PAGE_SIZE));
     }
 
-    RollupJobConfig(String id, String indexPattern, String rollupIndex, String cron, int pageSize, GroupConfig groupConfig,
-                    List<MetricConfig> metricsConfig, TimeValue timeout) {
+    public RollupJobConfig(final String id,
+                           final String indexPattern,
+                           final String rollupIndex,
+                           final String cron,
+                           final int pageSize,
+                           final GroupConfig groupConfig,
+                           final List<MetricConfig> metricsConfig,
+                           final @Nullable TimeValue timeout) {
+        if (id == null || id.isEmpty()) {
+            throw new IllegalArgumentException("Id must be a non-null, non-empty string");
+        }
+        if (indexPattern == null || indexPattern.isEmpty()) {
+            throw new IllegalArgumentException("Index pattern must be a non-null, non-empty string");
+        }
+        if (Regex.isMatchAllPattern(indexPattern)) {
+            throw new IllegalArgumentException("Index pattern must not match all indices (as it would match it's own rollup index");
+        }
+        if (Regex.isSimpleMatchPattern(indexPattern)) {
+            if (Regex.simpleMatch(indexPattern, rollupIndex)) {
+                throw new IllegalArgumentException("Index pattern would match rollup index name which is not allowed");
+            }
+        }
+        if (indexPattern.equals(rollupIndex)) {
+            throw new IllegalArgumentException("Rollup index may not be the same as the index pattern");
+        }
+        if (rollupIndex == null || rollupIndex.isEmpty()) {
+            throw new IllegalArgumentException("Rollup index must be a non-null, non-empty string");
+        }
+        if (cron == null || cron.isEmpty()) {
+            throw new IllegalArgumentException("Cron schedule must be a non-null, non-empty string");
+        }
+        if (pageSize <= 0) {
+            throw new IllegalArgumentException("Page size is mandatory and  must be a positive long");
+        }
+        // Cron doesn't have a parse helper method to see if the cron is valid,
+        // so just construct a temporary cron object and if the cron is bad, it'll
+        // throw an exception
+        Cron testCron = new Cron(cron);
+        if (groupConfig == null && (metricsConfig == null || metricsConfig.isEmpty())) {
+            throw new IllegalArgumentException("At least one grouping or metric must be configured");
+        }
+
         this.id = id;
         this.indexPattern = indexPattern;
         this.rollupIndex = rollupIndex;
         this.groupConfig = groupConfig;
-        this.metricsConfig = metricsConfig;
-        this.timeout = timeout;
+        this.metricsConfig = metricsConfig != null ? metricsConfig : Collections.emptyList();
+        this.timeout = timeout != null ? timeout : DEFAULT_TIMEOUT;
         this.cron = cron;
         this.pageSize = pageSize;
     }
 
-    public RollupJobConfig(StreamInput in) throws IOException {
+    public RollupJobConfig(final StreamInput in) throws IOException {
         id = in.readString();
         indexPattern = in.readString();
         rollupIndex = in.readString();
@@ -95,8 +145,6 @@ public RollupJobConfig(StreamInput in) throws IOException {
         pageSize = in.readInt();
     }
 
-    public RollupJobConfig() {}
-
     public String getId() {
         return id;
     }
@@ -135,13 +183,20 @@ public String getWriteableName() {
     }
 
     public Set<String> getAllFields() {
-        Set<String> fields = new HashSet<>(groupConfig.getAllFields());
-        fields.addAll(metricsConfig.stream().map(MetricConfig::getField).collect(Collectors.toSet()));
-        return fields;
+        final Set<String> fields = new HashSet<>();
+        if (groupConfig != null) {
+            fields.addAll(groupConfig.getAllFields());
+        }
+        if (metricsConfig != null) {
+            for (MetricConfig metric : metricsConfig) {
+                fields.add(metric.getField());
+            }
+        }
+        return Collections.unmodifiableSet(fields);
     }
 
-    public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCapsResponse,
-                                                             ActionRequestValidationException validationException) {
+    public void validateMappings(final Map<String, Map<String, FieldCapabilities>> fieldCapsResponse,
+                                 final ActionRequestValidationException validationException) {
         groupConfig.validateMappings(fieldCapsResponse, validationException);
         for (MetricConfig m : metricsConfig) {
             m.validateMappings(fieldCapsResponse, validationException);
@@ -149,34 +204,34 @@ public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCa
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
         builder.startObject();
-        builder.field(RollupField.ID.getPreferredName(), id);
-        builder.field(INDEX_PATTERN.getPreferredName(), indexPattern);
-        builder.field(ROLLUP_INDEX.getPreferredName(), rollupIndex);
-        builder.field(CRON.getPreferredName(), cron);
-        if (groupConfig != null) {
-            builder.field(GROUPS.getPreferredName(), groupConfig);
-        }
-        if (metricsConfig != null) {
-            builder.startArray(METRICS.getPreferredName());
-            for (MetricConfig config : metricsConfig) {
-                builder.startObject();
-                config.toXContent(builder, params);
-                builder.endObject();
+        {
+            builder.field(ID, id);
+            builder.field(INDEX_PATTERN, indexPattern);
+            builder.field(ROLLUP_INDEX, rollupIndex);
+            builder.field(CRON, cron);
+            if (groupConfig != null) {
+                builder.field(GroupConfig.NAME, groupConfig);
             }
-            builder.endArray();
-        }
-        if (timeout != null) {
-            builder.field(TIMEOUT.getPreferredName(), timeout);
+            if (metricsConfig != null) {
+                builder.startArray(MetricConfig.NAME);
+                for (MetricConfig metric : metricsConfig) {
+                    metric.toXContent(builder, params);
+                }
+                builder.endArray();
+            }
+            if (timeout != null) {
+                builder.field(TIMEOUT, timeout);
+            }
+            builder.field(PAGE_SIZE, pageSize);
         }
-        builder.field(PAGE_SIZE.getPreferredName(), pageSize);
         builder.endObject();
         return builder;
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    public void writeTo(final StreamOutput out) throws IOException {
         out.writeString(id);
         out.writeString(indexPattern);
         out.writeString(rollupIndex);
@@ -192,13 +247,11 @@ public boolean equals(Object other) {
         if (this == other) {
             return true;
         }
-
         if (other == null || getClass() != other.getClass()) {
             return false;
         }
 
-        RollupJobConfig that = (RollupJobConfig) other;
-
+        final RollupJobConfig that = (RollupJobConfig) other;
         return Objects.equals(this.id, that.id)
                 && Objects.equals(this.indexPattern, that.indexPattern)
                 && Objects.equals(this.rollupIndex, that.rollupIndex)
@@ -211,8 +264,7 @@ public boolean equals(Object other) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, indexPattern, rollupIndex, cron, groupConfig,
-                metricsConfig, timeout, pageSize);
+        return Objects.hash(id, indexPattern, rollupIndex, cron, groupConfig, metricsConfig, timeout, pageSize);
     }
 
     @Override
@@ -227,193 +279,7 @@ public String toJSONString() {
         return toString();
     }
 
-    public static class Builder implements Writeable, ToXContentObject {
-        private String id;
-        private String indexPattern;
-        private String rollupIndex;
-        private GroupConfig groupConfig;
-        private List<MetricConfig> metricsConfig = Collections.emptyList();
-        private TimeValue timeout = TimeValue.timeValueSeconds(20);
-        private String cron;
-        private int pageSize = 0;
-
-        public Builder(RollupJobConfig job) {
-            this.id = job.getId();
-            this.indexPattern = job.getIndexPattern();
-            this.rollupIndex = job.getRollupIndex();
-            this.groupConfig = job.getGroupConfig();
-            this.metricsConfig = job.getMetricsConfig();
-            this.timeout = job.getTimeout();
-            this.cron = job.getCron();
-            this.pageSize = job.getPageSize();
-        }
-
-        public static RollupJobConfig.Builder fromXContent(String id, XContentParser parser) {
-            RollupJobConfig.Builder config = RollupJobConfig.PARSER.apply(parser, null);
-            if (id != null) {
-                config.setId(id);
-            }
-            return config;
-        }
-
-        public Builder() {}
-
-        public String getId() {
-            return id;
-        }
-
-        public RollupJobConfig.Builder setId(String id) {
-            this.id = id;
-            return this;
-        }
-
-        public String getIndexPattern() {
-            return indexPattern;
-        }
-
-        public RollupJobConfig.Builder setIndexPattern(String indexPattern) {
-            this.indexPattern = indexPattern;
-            return this;
-        }
-
-        public String getRollupIndex() {
-            return rollupIndex;
-        }
-
-        public RollupJobConfig.Builder setRollupIndex(String rollupIndex) {
-            this.rollupIndex = rollupIndex;
-            return this;
-        }
-
-        public GroupConfig getGroupConfig() {
-            return groupConfig;
-        }
-
-        public RollupJobConfig.Builder setGroupConfig(GroupConfig groupConfig) {
-            this.groupConfig = groupConfig;
-            return this;
-        }
-
-        public List<MetricConfig> getMetricsConfig() {
-            return metricsConfig;
-        }
-
-        public RollupJobConfig.Builder setMetricsConfig(List<MetricConfig> metricsConfig) {
-            this.metricsConfig = metricsConfig;
-            return this;
-        }
-
-        public TimeValue getTimeout() {
-            return timeout;
-        }
-
-        public RollupJobConfig.Builder setTimeout(TimeValue timeout) {
-            this.timeout = timeout;
-            return this;
-        }
-
-        public String getCron() {
-            return cron;
-        }
-
-        public RollupJobConfig.Builder setCron(String cron) {
-            this.cron = cron;
-            return this;
-        }
-
-        public int getPageSize() {
-            return pageSize;
-        }
-
-        public RollupJobConfig.Builder setPageSize(int pageSize) {
-            this.pageSize = pageSize;
-            return this;
-        }
-
-        public RollupJobConfig build() {
-            if (id == null || id.isEmpty()) {
-                throw new IllegalArgumentException("An ID is mandatory.");
-            }
-            if (indexPattern == null || indexPattern.isEmpty()) {
-                throw new IllegalArgumentException("An index pattern is mandatory.");
-            }
-            if (Regex.isMatchAllPattern(indexPattern)) {
-                throw new IllegalArgumentException("Index pattern must not match all indices (as it would match it's own rollup index");
-            }
-            if (Regex.isSimpleMatchPattern(indexPattern)) {
-                if (Regex.simpleMatch(indexPattern, rollupIndex)) {
-                    throw new IllegalArgumentException("Index pattern would match rollup index name which is not allowed.");
-                }
-            }
-            if (indexPattern.equals(rollupIndex)) {
-                throw new IllegalArgumentException("Rollup index may not be the same as the index pattern.");
-            }
-            if (rollupIndex == null || rollupIndex.isEmpty()) {
-                throw new IllegalArgumentException("A rollup index name is mandatory.");
-            }
-            if (cron == null || cron.isEmpty()) {
-                throw new IllegalArgumentException("A cron schedule is mandatory.");
-            }
-            if (pageSize <= 0) {
-                throw new IllegalArgumentException("Parameter [" + PAGE_SIZE.getPreferredName()
-                        + "] is mandatory and  must be a positive long.");
-            }
-            // Cron doesn't have a parse helper method to see if the cron is valid,
-            // so just construct a temporary cron object and if the cron is bad, it'll
-            // throw an exception
-            Cron testCron = new Cron(cron);
-            if (groupConfig == null && (metricsConfig == null || metricsConfig.isEmpty())) {
-                throw new IllegalArgumentException("At least one grouping or metric must be configured.");
-            }
-            return new RollupJobConfig(id, indexPattern, rollupIndex, cron, pageSize, groupConfig,
-                    metricsConfig, timeout);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            if (id != null) {
-                builder.field(RollupField.ID.getPreferredName(), id);
-            }
-            if (indexPattern != null) {
-                builder.field(INDEX_PATTERN.getPreferredName(), indexPattern);
-            }
-            if (indexPattern != null) {
-                builder.field(ROLLUP_INDEX.getPreferredName(), rollupIndex);
-            }
-            if (cron != null) {
-                builder.field(CRON.getPreferredName(), cron);
-            }
-            if (groupConfig != null) {
-                builder.field(GROUPS.getPreferredName(), groupConfig);
-            }
-            if (metricsConfig != null) {
-                builder.startArray(METRICS.getPreferredName());
-                for (MetricConfig config : metricsConfig) {
-                    builder.startObject();
-                    config.toXContent(builder, params);
-                    builder.endObject();
-                }
-                builder.endArray();
-            }
-            if (timeout != null) {
-                builder.field(TIMEOUT.getPreferredName(), timeout);
-            }
-            builder.field(PAGE_SIZE.getPreferredName(), pageSize);
-            builder.endObject();
-            return builder;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(id);
-            out.writeOptionalString(indexPattern);
-            out.writeOptionalString(rollupIndex);
-            out.writeOptionalString(cron);
-            out.writeOptionalWriteable(groupConfig);
-            out.writeList(metricsConfig);
-            out.writeTimeValue(timeout);
-            out.writeInt(pageSize);
-        }
+    public static RollupJobConfig fromXContent(final XContentParser parser, @Nullable final String optionalJobId) throws IOException {
+        return PARSER.parse(parser, optionalJobId);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java
index 4cbd5a3b4559a..640385c9c80d5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.rollup.job;
 
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -39,12 +40,19 @@ public class RollupJobStatus implements Task.Status, PersistentTaskState {
     @Nullable
     private final TreeMap<String, Object> currentPosition;
 
+    // Flag holds the state of the ID scheme, e.g. if it has been upgraded to the
+    // concatenation scheme.  See #32372 for more details
+    private boolean upgradedDocumentID;
+
     private static final ParseField STATE = new ParseField("job_state");
     private static final ParseField CURRENT_POSITION = new ParseField("current_position");
+    private static final ParseField UPGRADED_DOC_ID = new ParseField("upgraded_doc_id");
 
     public static final ConstructingObjectParser<RollupJobStatus, Void> PARSER =
             new ConstructingObjectParser<>(NAME,
-                    args -> new RollupJobStatus((IndexerState) args[0], (HashMap<String, Object>) args[1]));
+                    args -> new RollupJobStatus((IndexerState) args[0],
+                        (HashMap<String, Object>) args[1],
+                        (Boolean)args[2]));
 
     static {
         PARSER.declareField(constructorArg(), p -> {
@@ -62,16 +70,28 @@ public class RollupJobStatus implements Task.Status, PersistentTaskState {
             }
             throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
         }, CURRENT_POSITION, ObjectParser.ValueType.VALUE_OBJECT_ARRAY);
+
+        // Optional to accommodate old versions of state
+        PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), UPGRADED_DOC_ID);
     }
 
-    public RollupJobStatus(IndexerState state, @Nullable Map<String, Object> position) {
+    public RollupJobStatus(IndexerState state, @Nullable Map<String, Object> position,
+                           @Nullable Boolean upgradedDocumentID) {
         this.state = state;
         this.currentPosition = position == null ? null : new TreeMap<>(position);
+        this.upgradedDocumentID = upgradedDocumentID != null ? upgradedDocumentID : false;  //default to false if missing
     }
 
     public RollupJobStatus(StreamInput in) throws IOException {
         state = IndexerState.fromStream(in);
         currentPosition = in.readBoolean() ? new TreeMap<>(in.readMap()) : null;
+        if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
+            upgradedDocumentID = in.readBoolean();
+        } else {
+            // If we're getting this job from a pre-6.4.0 node,
+            // it is using the old ID scheme
+            upgradedDocumentID = false;
+        }
     }
 
     public IndexerState getIndexerState() {
@@ -82,6 +102,10 @@ public Map<String, Object> getPosition() {
         return currentPosition;
     }
 
+    public boolean isUpgradedDocumentID() {
+        return upgradedDocumentID;
+    }
+
     public static RollupJobStatus fromXContent(XContentParser parser) {
         try {
             return PARSER.parse(parser, null);
@@ -97,6 +121,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (currentPosition != null) {
             builder.field(CURRENT_POSITION.getPreferredName(), currentPosition);
         }
+        builder.field(UPGRADED_DOC_ID.getPreferredName(), upgradedDocumentID);
         builder.endObject();
         return builder;
     }
@@ -113,6 +138,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (currentPosition != null) {
             out.writeMap(currentPosition);
         }
+        if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
+            out.writeBoolean(upgradedDocumentID);
+        }
     }
 
     @Override
@@ -128,11 +156,12 @@ public boolean equals(Object other) {
         RollupJobStatus that = (RollupJobStatus) other;
 
         return Objects.equals(this.state, that.state)
-                && Objects.equals(this.currentPosition, that.currentPosition);
+            && Objects.equals(this.currentPosition, that.currentPosition)
+            && Objects.equals(this.upgradedDocumentID, that.upgradedDocumentID);
     }
 
     @Override
     public int hashCode() {
-    return Objects.hash(state, currentPosition);
+    return Objects.hash(state, currentPosition, upgradedDocumentID);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java
index da73020f0087f..32507d57f32b0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java
@@ -12,9 +12,10 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder;
@@ -24,13 +25,13 @@
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
 /**
  * The configuration object for the histograms in the rollup config
  *
@@ -42,20 +43,29 @@
  *     ]
  * }
  */
-public class TermsGroupConfig implements Writeable, ToXContentFragment {
-    private static final String NAME = "term_group_config";
-    public static final ObjectParser<TermsGroupConfig.Builder, Void> PARSER = new ObjectParser<>(NAME, TermsGroupConfig.Builder::new);
+public class TermsGroupConfig implements Writeable, ToXContentObject {
+
+    static final String NAME = "terms";
+    private static final String FIELDS = "fields";
 
-    private static final ParseField FIELDS = new ParseField("fields");
     private static final List<String> FLOAT_TYPES = Arrays.asList("half_float", "float", "double", "scaled_float");
     private static final List<String> NATURAL_TYPES = Arrays.asList("byte", "short", "integer", "long");
-    private final String[] fields;
 
+    private static final ConstructingObjectParser<TermsGroupConfig, Void> PARSER;
     static {
-        PARSER.declareStringArray(TermsGroupConfig.Builder::setFields, FIELDS);
+        PARSER = new ConstructingObjectParser<>(NAME, args -> {
+            @SuppressWarnings("unchecked") List<String> fields = (List<String>) args[0];
+            return new TermsGroupConfig(fields != null ? fields.toArray(new String[fields.size()]) : null);
+        });
+        PARSER.declareStringArray(constructorArg(), new ParseField(FIELDS));
     }
 
-    private TermsGroupConfig(String[] fields) {
+    private final String[] fields;
+
+    public TermsGroupConfig(final String... fields) {
+        if (fields == null || fields.length == 0) {
+            throw new IllegalArgumentException("Fields must have at least one value");
+        }
         this.fields = fields;
     }
 
@@ -63,6 +73,9 @@ private TermsGroupConfig(String[] fields) {
         fields = in.readStringArray();
     }
 
+    /**
+     * @return the names of the fields. Never {@code null}.
+     */
     public String[] getFields() {
         return fields;
     }
@@ -72,10 +85,6 @@ public String[] getFields() {
      * set of date histograms.  Used by the rollup indexer to iterate over historical data
      */
     public List<CompositeValuesSourceBuilder<?>> toBuilders() {
-        if (fields.length == 0) {
-            return Collections.emptyList();
-        }
-
         return Arrays.stream(fields).map(f -> {
             TermsValuesSourceBuilder vsBuilder
                     = new TermsValuesSourceBuilder(RollupField.formatIndexerAggName(f, TermsAggregationBuilder.NAME));
@@ -94,14 +103,6 @@ public Map<String, Object> toAggCap() {
         return map;
     }
 
-    public Map<String, Object> getMetadata() {
-        return Collections.emptyMap();
-    }
-
-    public Set<String> getAllFields() {
-        return Arrays.stream(fields).collect(Collectors.toSet());
-    }
-
     public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCapsResponse,
                                  ActionRequestValidationException validationException) {
 
@@ -138,8 +139,11 @@ public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCa
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(FIELDS.getPreferredName(), fields);
-        return builder;
+        builder.startObject();
+        {
+            builder.field(FIELDS, fields);
+        }
+        return builder.endObject();
     }
 
     @Override
@@ -148,18 +152,15 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public boolean equals(Object other) {
+    public boolean equals(final Object other) {
         if (this == other) {
             return true;
         }
-
         if (other == null || getClass() != other.getClass()) {
             return false;
         }
-
-        TermsGroupConfig that = (TermsGroupConfig) other;
-
-        return Arrays.equals(this.fields, that.fields);
+        final TermsGroupConfig that = (TermsGroupConfig) other;
+        return Arrays.equals(fields, that.fields);
     }
 
     @Override
@@ -172,23 +173,7 @@ public String toString() {
         return Strings.toString(this, true, true);
     }
 
-    public static class Builder {
-        private List<String> fields;
-
-        public List<String> getFields() {
-            return fields;
-        }
-
-        public TermsGroupConfig.Builder setFields(List<String> fields) {
-            this.fields = fields;
-            return this;
-        }
-
-        public TermsGroupConfig build() {
-            if (fields == null || fields.isEmpty()) {
-                throw new IllegalArgumentException("Parameter [" + FIELDS + "] must have at least one value.");
-            }
-            return new TermsGroupConfig(fields.toArray(new String[0]));
-        }
+    public static TermsGroupConfig fromXContent(final XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
index 99788ac1de43a..8d56221d78bee 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.io.IOException;
 import java.util.Objects;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/UserSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/UserSettings.java
index 7f22f90351ef5..536464cb3376d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/UserSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/UserSettings.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.io.IOException;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
index 559e0ab8d9877..585fb372d3247 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
@@ -50,7 +50,7 @@ public String application() {
 
     @Override
     public Collection<String> getApplicationNames() {
-        return Collections.singleton(application);
+        return application == null ? Collections.emptySet() : Collections.singleton(application);
     }
 
     public void privileges(String... privileges) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateResponse.java
index 0cf7ace1103d0..c3e126b92ce11 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateResponse.java
@@ -8,7 +8,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.io.IOException;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java
index d7538c2a556f0..05b3af41de222 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.support.Validation;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.xcontent.XContentUtils;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java
index 666b79cfe5db7..0da525b6ffc4f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java
@@ -8,7 +8,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.io.IOException;
 import java.util.Collection;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java
index eea804d81fe9f..7dc958bbef9f9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.support.Validation;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.xcontent.XContentUtils;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
index 161d9d449990f..e72df007cf63c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xpack.core.security.user.InternalUserSerializationHelper;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.io.IOException;
 import java.util.Base64;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java
index 0f073ef4ae398..08e01025e7e84 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java
@@ -6,7 +6,7 @@
 package org.elasticsearch.xpack.core.security.authc;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.util.Objects;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java
index 2c63ca95eb980..ae04d474f41cb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xpack.core.XPackField;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.util.Collections;
 import java.util.HashMap;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
index 42bd771103fdd..54fd8cc7974b2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
-import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 import org.elasticsearch.xpack.core.security.support.Validation;
 import org.elasticsearch.xpack.core.security.xcontent.XContentUtils;
 
@@ -163,7 +162,7 @@ public String toString() {
         }
         sb.append("], runAs=[").append(Strings.arrayToCommaDelimitedString(runAs));
         sb.append("], metadata=[");
-        MetadataUtils.writeValue(sb, metadata);
+        sb.append(metadata);
         sb.append("]]");
         return sb.toString();
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java
index f945b4e24c77c..e812f0cfc7332 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java
@@ -50,6 +50,7 @@
 import org.elasticsearch.index.query.Rewriteable;
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.elasticsearch.index.search.NestedHelper;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
@@ -62,7 +63,7 @@
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -73,6 +74,7 @@
 import java.util.Map;
 import java.util.function.Function;
 
+import static org.apache.lucene.search.BooleanClause.Occur.FILTER;
 import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
 
 /**
@@ -139,6 +141,13 @@ protected DirectoryReader wrap(DirectoryReader reader) {
                         Query roleQuery = queryShardContext.toFilter(queryBuilder).query();
                         filter.add(roleQuery, SHOULD);
                         if (queryShardContext.getMapperService().hasNested()) {
+                            NestedHelper nestedHelper = new NestedHelper(queryShardContext.getMapperService());
+                            if (nestedHelper.mightMatchNestedDocs(roleQuery)) {
+                                roleQuery = new BooleanQuery.Builder()
+                                    .add(roleQuery, FILTER)
+                                    .add(Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()), FILTER)
+                                    .build();
+                            }
                             // If access is allowed on root doc then also access is allowed on all nested docs of that root document:
                             BitSetProducer rootDocs = queryShardContext.bitsetFilter(
                                     Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java
index c068c77781b14..e204d89b1c095 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java
@@ -135,7 +135,9 @@ public ManageApplicationPrivileges(Set<String> applicationNames) {
             this.requestPredicate = request -> {
                 if (request instanceof ApplicationPrivilegesRequest) {
                     final ApplicationPrivilegesRequest privRequest = (ApplicationPrivilegesRequest) request;
-                    return privRequest.getApplicationNames().stream().allMatch(application -> applicationPredicate.test(application));
+                    final Collection<String> requestApplicationNames = privRequest.getApplicationNames();
+                    return requestApplicationNames.isEmpty() ? this.applicationNames.contains("*")
+                        : requestApplicationNames.stream().allMatch(application -> applicationPredicate.test(application));
                 }
                 return false;
             };
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java
index ff457cb8d06ec..a48f562c7af7b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java
@@ -5,8 +5,6 @@
  */
 package org.elasticsearch.xpack.core.security.support;
 
-import java.lang.reflect.Array;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
 
@@ -19,50 +17,6 @@ public class MetadataUtils {
     private MetadataUtils() {
     }
 
-    public static void writeValue(StringBuilder sb, Object object) {
-        if (object == null) {
-            sb.append(object);
-        } else if (object instanceof Map) {
-            sb.append("{");
-            for (Map.Entry<String, Object> entry : ((Map<String, Object>) object).entrySet()) {
-                sb.append(entry.getKey()).append("=");
-                writeValue(sb, entry.getValue());
-            }
-            sb.append("}");
-
-        } else if (object instanceof Collection) {
-            sb.append("[");
-            boolean first = true;
-            for (Object item : (Collection) object) {
-                if (!first) {
-                    sb.append(",");
-                }
-                writeValue(sb, item);
-                first = false;
-            }
-            sb.append("]");
-        } else if (object.getClass().isArray()) {
-            sb.append("[");
-            for (int i = 0; i < Array.getLength(object); i++) {
-                if (i != 0) {
-                    sb.append(",");
-                }
-                writeValue(sb, Array.get(object, i));
-            }
-            sb.append("]");
-        } else {
-            sb.append(object);
-        }
-    }
-
-    public static void verifyNoReservedMetadata(Map<String, Object> metadata) {
-        for (String key : metadata.keySet()) {
-            if (key.startsWith(RESERVED_PREFIX)) {
-                throw new IllegalArgumentException("invalid user metadata. [" + key + "] is a reserved for internal use");
-            }
-        }
-    }
-
     public static boolean containsReservedMetadata(Map<String, Object> metadata) {
         for (String key : metadata.keySet()) {
             if (key.startsWith(RESERVED_PREFIX)) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java
index 36354ff58b318..ae6f41c3a1bd4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 
 import java.util.Collections;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java
index dfa437fa8d2c4..9db64da97a539 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.security.user;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 
 /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java
index ec618a4f4821c..c58f86ea42245 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.user;
 
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUserSerializationHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUserSerializationHelper.java
index fa41828a7bba8..c0b45aea57c20 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUserSerializationHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUserSerializationHelper.java
@@ -7,6 +7,7 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.io.IOException;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java
index 8dfa149987d0e..3e816aa54bca5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.user;
 
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 
 /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java
index ce37d742a191f..047758177fb0b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.security.user;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 
 /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java
index 4569c2a68a09b..1c7ac129d17b8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.user;
 
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.authz.privilege.SystemPrivilege;
 
 import java.util.function.Predicate;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackSecurityUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackSecurityUser.java
index 906d354837783..e98df7fb50a56 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackSecurityUser.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackSecurityUser.java
@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.xpack.core.security.user;
 
+import org.elasticsearch.protocol.xpack.security.User;
+
 /**
  * internal user that manages xpack security. Has all cluster/indices permissions.
  */
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java
index 38c9fe84aa934..fe50b1b9c88bc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.user;
 
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.index.IndexAuditTrailField;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeInfoAction.java
index f17dfbdb90b9b..3044c953a3e09 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeInfoAction.java
@@ -6,27 +6,13 @@
 package org.elasticsearch.xpack.core.upgrade.actions;
 
 import org.elasticsearch.action.Action;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.core.upgrade.UpgradeActionRequired;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoResponse;
 
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-
-public class IndexUpgradeInfoAction extends Action<IndexUpgradeInfoAction.Response> {
+public class IndexUpgradeInfoAction extends Action<IndexUpgradeInfoResponse> {
 
     public static final IndexUpgradeInfoAction INSTANCE = new IndexUpgradeInfoAction();
     public static final String NAME = "cluster:admin/xpack/upgrade/info";
@@ -36,149 +22,15 @@ private IndexUpgradeInfoAction() {
     }
 
     @Override
-    public Response newResponse() {
-        return new Response();
-    }
-
-    public static class Response extends ActionResponse implements ToXContentObject {
-        private Map<String, UpgradeActionRequired> actions;
-
-        public Response() {
-
-        }
-
-        public Response(Map<String, UpgradeActionRequired> actions) {
-            this.actions = actions;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            actions = in.readMap(StreamInput::readString, UpgradeActionRequired::readFromStream);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeMap(actions, StreamOutput::writeString, (out1, value) -> value.writeTo(out1));
-        }
-
-        public Map<String, UpgradeActionRequired> getActions() {
-            return actions;
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            {
-                builder.startObject("indices");
-                for (Map.Entry<String, UpgradeActionRequired> entry : actions.entrySet()) {
-                    builder.startObject(entry.getKey());
-                    {
-                        builder.field("action_required", entry.getValue().toString());
-                    }
-                    builder.endObject();
-                }
-                builder.endObject();
-            }
-            builder.endObject();
-            return builder;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            Response response = (Response) o;
-            return Objects.equals(actions, response.actions);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(actions);
-        }
+    public IndexUpgradeInfoResponse newResponse() {
+        return new IndexUpgradeInfoResponse();
     }
 
-    public static class Request extends MasterNodeReadRequest<Request> implements IndicesRequest.Replaceable {
-
-        private String[] indices = null;
-        private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true);
-
-        // for serialization
-        public Request() {
-
-        }
-
-        public Request(String... indices) {
-            this.indices = indices;
-        }
-
-        public Request(StreamInput in) throws IOException {
-            super(in);
-            indices = in.readStringArray();
-            indicesOptions = IndicesOptions.readIndicesOptions(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeStringArray(indices);
-            indicesOptions.writeIndicesOptions(out);
-        }
-
-        @Override
-        public String[] indices() {
-            return indices;
-        }
-
-        @Override
-        public Request indices(String... indices) {
-            this.indices = indices;
-            return this;
-        }
-
-        @Override
-        public IndicesOptions indicesOptions() {
-            return indicesOptions;
-        }
-
-        public void indicesOptions(IndicesOptions indicesOptions) {
-            this.indicesOptions = indicesOptions;
-        }
-
-        @Override
-        public ActionRequestValidationException validate() {
-            ActionRequestValidationException validationException = null;
-            if (indices == null) {
-                validationException = addValidationError("index/indices is missing", validationException);
-            }
-            return validationException;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            Request request = (Request) o;
-            return Arrays.equals(indices, request.indices) &&
-                    Objects.equals(indicesOptions.toString(), request.indicesOptions.toString());
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(Arrays.hashCode(indices), indicesOptions.toString());
-        }
-    }
-
-    public static class RequestBuilder extends MasterNodeReadOperationRequestBuilder<Request, Response, RequestBuilder> {
+    public static class RequestBuilder
+        extends MasterNodeReadOperationRequestBuilder<IndexUpgradeInfoRequest, IndexUpgradeInfoResponse, RequestBuilder> {
 
         public RequestBuilder(ElasticsearchClient client) {
-            super(client, INSTANCE, new Request());
+            super(client, INSTANCE, new IndexUpgradeInfoRequest());
         }
 
         public RequestBuilder setIndices(String... indices) {
@@ -191,5 +43,4 @@ public RequestBuilder setIndicesOptions(IndicesOptions indicesOptions) {
             return this;
         }
     }
-
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
index 063f1f655a46e..b9458c23ddc74 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
@@ -19,9 +19,9 @@
 import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java
index 8a16755a6dbce..eb440ddc2510e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.watcher.transport.actions.delete;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 
 /**
  * This action deletes an watch from in memory, the scheduler and the index
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java
index afea505aa4a91..6779eec1ddf73 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java
@@ -7,6 +7,8 @@
 
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 
 /**
  * A delete document action request builder.
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchResponse.java
deleted file mode 100644
index bda5ecabaa022..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchResponse.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.watcher.transport.actions.delete;
-
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-
-import java.io.IOException;
-
-public class DeleteWatchResponse extends ActionResponse {
-
-    private String id;
-    private long version;
-    private boolean found;
-
-    public DeleteWatchResponse() {
-    }
-
-    public DeleteWatchResponse(String id, long version, boolean found) {
-        this.id = id;
-        this.version = version;
-        this.found = found;
-    }
-
-    public String getId() {
-        return id;
-    }
-
-    public long getVersion() {
-        return version;
-    }
-
-    public boolean isFound() {
-        return found;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        id = in.readString();
-        version = in.readVLong();
-        found = in.readBoolean();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeString(id);
-        out.writeVLong(version);
-        out.writeBoolean(found);
-    }
-}
diff --git a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
index 0cd7a32bcc47b..70abb67a76240 100644
--- a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
@@ -15,6 +15,8 @@ grant {
 grant codeBase "${codebase.netty-common}" {
    // for reading the system-wide configuration for the backlog of established sockets
    permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };
 
 grant codeBase "${codebase.netty-transport}" {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseFIPSTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseFIPSTests.java
new file mode 100644
index 0000000000000..c432a207fcb70
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseFIPSTests.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.license;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.verify;
+
+public class LicenseFIPSTests extends AbstractLicenseServiceTestCase {
+
+    public void testFIPSCheckWithAllowedLicense() throws Exception {
+        License newLicense = TestUtils.generateSignedLicense(randomFrom("trial", "platinum"), TimeValue.timeValueHours(24L));
+        PutLicenseRequest request = new PutLicenseRequest();
+        request.acknowledge(true);
+        request.license(newLicense);
+        Settings settings = Settings.builder()
+            .put("xpack.security.enabled", true)
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.security.fips_mode.enabled", randomBoolean())
+            .build();
+        XPackLicenseState licenseState = new XPackLicenseState(settings);
+
+        setInitialState(null, licenseState, settings);
+        licenseService.start();
+        PlainActionFuture<PutLicenseResponse> responseFuture = new PlainActionFuture<>();
+        licenseService.registerLicense(request, responseFuture);
+        verify(clusterService).submitStateUpdateTask(any(String.class), any(ClusterStateUpdateTask.class));
+    }
+
+    public void testFIPSCheckWithoutAllowedLicense() throws Exception {
+        License newLicense = TestUtils.generateSignedLicense(randomFrom("gold", "standard"), TimeValue.timeValueHours(24L));
+        PutLicenseRequest request = new PutLicenseRequest();
+        request.acknowledge(true);
+        request.license(newLicense);
+        Settings settings = Settings.builder()
+            .put("xpack.security.enabled", true)
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.security.fips_mode.enabled", true)
+            .build();
+        XPackLicenseState licenseState = new XPackLicenseState(settings);
+
+        setInitialState(null, licenseState, settings);
+        licenseService.start();
+        PlainActionFuture<PutLicenseResponse> responseFuture = new PlainActionFuture<>();
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> licenseService.registerLicense(request, responseFuture));
+        assertThat(e.getMessage(),
+            containsString("Cannot install a [" + newLicense.operationMode() + "] license unless FIPS mode is disabled"));
+        licenseService.stop();
+
+        settings = Settings.builder()
+            .put("xpack.security.enabled", true)
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.security.fips_mode.enabled", false)
+            .build();
+        licenseState = new XPackLicenseState(settings);
+
+        setInitialState(null, licenseState, settings);
+        licenseService.start();
+        licenseService.registerLicense(request, responseFuture);
+        verify(clusterService).submitStateUpdateTask(any(String.class), any(ClusterStateUpdateTask.class));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java
index c397bd79e2885..f3fbab1026efd 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.LicensesStatus;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
index aa372eb03562a..82fde59a8ae6b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseResponse;
 import org.elasticsearch.protocol.xpack.license.LicensesStatus;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 import org.elasticsearch.test.ESSingleNodeTestCase;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
index 796cae375e3a6..e0029362003df 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
@@ -349,13 +349,6 @@ public void onIndexModule(IndexModule indexModule) {
         filterPlugins(Plugin.class).stream().forEach(p -> p.onIndexModule(indexModule));
     }
 
-    @Override
-    public Map<String, Supplier<ClusterState.Custom>> getInitialClusterStateCustomSupplier() {
-        Map<String, Supplier<ClusterState.Custom>> suppliers = new HashMap<>();
-        filterPlugins(ClusterPlugin.class).stream().forEach(p -> suppliers.putAll(p.getInitialClusterStateCustomSupplier()));
-        return suppliers;
-    }
-
     @Override
     public Function<String, Predicate<String>> getFieldFilter() {
         List<Function<String, Predicate<String>>> items = filterPlugins(MapperPlugin.class).stream().map(p ->
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java
index 00a1586c1d570..50c60a31427d6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java
@@ -87,14 +87,6 @@ public static AnalysisConfig.Builder createRandomized() {
         if (randomBoolean()) {
             builder.setLatency(TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000)));
         }
-        if (randomBoolean()) {
-            int numBucketSpans = randomIntBetween(0, 10);
-            List<TimeValue> multipleBucketSpans = new ArrayList<>();
-            for (int i = 2; i <= numBucketSpans; i++) {
-                multipleBucketSpans.add(TimeValue.timeValueSeconds(bucketSpan.getSeconds() * i));
-            }
-            builder.setMultipleBucketSpans(multipleBucketSpans);
-        }
         if (randomBoolean()) {
             builder.setMultivariateByFields(randomBoolean());
         }
@@ -255,28 +247,6 @@ public void testFieldConfiguration_multipleDetectors_NotPreSummarised() {
         }
     }
 
-    public void testFieldConfiguration_singleDetector_PreSummarised() {
-        // Multiple detectors, pre-summarised
-        AnalysisConfig.Builder builder = createConfigBuilder();
-        builder.setSummaryCountFieldName("summaryCount");
-        AnalysisConfig ac = builder.build();
-
-        assertTrue(ac.analysisFields().contains("summaryCount"));
-        assertEquals("summaryCount", ac.getSummaryCountFieldName());
-
-        builder = createConfigBuilder();
-        builder.setBucketSpan(TimeValue.timeValueSeconds(1000));
-        builder.setMultipleBucketSpans(Arrays.asList(
-                TimeValue.timeValueSeconds(5000), TimeValue.timeValueSeconds(10000), TimeValue.timeValueSeconds(24000)));
-        ac = builder.build();
-        assertTrue(ac.getMultipleBucketSpans().contains(TimeValue.timeValueSeconds(5000)));
-        assertTrue(ac.getMultipleBucketSpans().contains(TimeValue.timeValueSeconds(10000)));
-        assertTrue(ac.getMultipleBucketSpans().contains(TimeValue.timeValueSeconds(24000)));
-
-        assertEquals(1, ac.getDetectors().size());
-        assertEquals(0, ac.getDetectors().get(0).getDetectorIndex());
-    }
-
     public void testBuild_GivenMlCategoryUsedAsByFieldButNoCategorizationFieldName() {
         Detector.Builder detector = new Detector.Builder();
         detector.setFunction("count");
@@ -693,58 +663,6 @@ public void testVerify_GivenMetricAndSummaryCountField() {
         assertEquals(Messages.getMessage(Messages.JOB_CONFIG_FUNCTION_INCOMPATIBLE_PRESUMMARIZED, DetectorFunction.METRIC), e.getMessage());
     }
 
-    public void testMultipleBucketsConfig() {
-        AnalysisConfig.Builder ac = createValidConfig();
-        ac.setMultipleBucketSpans(Arrays.asList(
-                TimeValue.timeValueSeconds(10L),
-                TimeValue.timeValueSeconds(15L),
-                TimeValue.timeValueSeconds(20L),
-                TimeValue.timeValueSeconds(25L),
-                TimeValue.timeValueSeconds(30L),
-                TimeValue.timeValueSeconds(35L)));
-        List<Detector> detectors = new ArrayList<>();
-        Detector detector = new Detector.Builder("count", null).build();
-        detectors.add(detector);
-        ac.setDetectors(detectors);
-
-        ac.setBucketSpan(TimeValue.timeValueSeconds(4L));
-        ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, ac::build);
-        assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, "10s", "4s"), e.getMessage());
-
-        ac.setBucketSpan(TimeValue.timeValueSeconds(5L));
-        ac.build();
-
-        AnalysisConfig.Builder ac2 = createValidConfig();
-        ac2.setBucketSpan(TimeValue.timeValueSeconds(5L));
-        ac2.setDetectors(detectors);
-        ac2.setMultipleBucketSpans(Arrays.asList(
-                TimeValue.timeValueSeconds(10L),
-                TimeValue.timeValueSeconds(15L),
-                TimeValue.timeValueSeconds(20L),
-                TimeValue.timeValueSeconds(25L),
-                TimeValue.timeValueSeconds(30L)));
-        assertFalse(ac.equals(ac2));
-        ac2.setMultipleBucketSpans(Arrays.asList(
-                TimeValue.timeValueSeconds(10L),
-                TimeValue.timeValueSeconds(15L),
-                TimeValue.timeValueSeconds(20L),
-                TimeValue.timeValueSeconds(25L),
-                TimeValue.timeValueSeconds(30L),
-                TimeValue.timeValueSeconds(35L)));
-
-        ac.setBucketSpan(TimeValue.timeValueSeconds(222L));
-        ac.setMultipleBucketSpans(Collections.emptyList());
-        ac.build();
-
-        ac.setMultipleBucketSpans(Collections.singletonList(TimeValue.timeValueSeconds(222L)));
-        e = ESTestCase.expectThrows(ElasticsearchException.class, ac::build);
-        assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, "3.7m", "3.7m"), e.getMessage());
-
-        ac.setMultipleBucketSpans(Arrays.asList(TimeValue.timeValueSeconds(-444L), TimeValue.timeValueSeconds(-888L)));
-        e = ESTestCase.expectThrows(ElasticsearchException.class, ac::build);
-        assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, -444, "3.7m"), e.getMessage());
-    }
-
     public void testVerify_GivenCategorizationFiltersButNoCategorizationFieldName() {
         AnalysisConfig.Builder config = createValidConfig();
         config.setCategorizationFilters(Collections.singletonList("foo"));
@@ -838,7 +756,7 @@ private static AnalysisConfig.Builder createValidCategorizationConfig() {
     @Override
     protected AnalysisConfig mutateInstance(AnalysisConfig instance) {
         AnalysisConfig.Builder builder = new AnalysisConfig.Builder(instance);
-        switch (between(0, 12)) {
+        switch (between(0, 11)) {
         case 0:
             List<Detector> detectors = new ArrayList<>(instance.getDetectors());
             Detector.Builder detector = new Detector.Builder();
@@ -849,7 +767,6 @@ protected AnalysisConfig mutateInstance(AnalysisConfig instance) {
             break;
         case 1:
             builder.setBucketSpan(new TimeValue(instance.getBucketSpan().millis() + (between(1, 1000) * 1000)));
-            builder.setMultipleBucketSpans(Collections.emptyList());
             break;
         case 2:
             if (instance.getLatency() == null) {
@@ -939,16 +856,6 @@ protected AnalysisConfig mutateInstance(AnalysisConfig instance) {
             }
             break;
         case 11:
-            List<TimeValue> multipleBucketSpans;
-            if (instance.getMultipleBucketSpans() == null) {
-                multipleBucketSpans = new ArrayList<>();
-            } else {
-                multipleBucketSpans = new ArrayList<>(instance.getMultipleBucketSpans());
-            }
-            multipleBucketSpans.add(new TimeValue(between(2, 10) * instance.getBucketSpan().millis()));
-            builder.setMultipleBucketSpans(multipleBucketSpans);
-            break;
-        case 12:
             boolean usePerPartitionNormalization = instance.getUsePerPartitionNormalization() == false;
             builder.setUsePerPartitionNormalization(usePerPartitionNormalization);
             if (usePerPartitionNormalization) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java
index 762d1d7ec3931..2f7eab0e97c70 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java
@@ -541,7 +541,7 @@ public void testVerify_GivenRulesAndFunctionIsLatLong() {
 
         ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build);
 
-        assertThat(e.getMessage(), equalTo("Invalid detector rule: function lat_long does not support rules with conditions"));
+        assertThat(e.getMessage(), equalTo("Invalid detector rule: function lat_long only supports conditions that apply to time"));
     }
 
     public void testVerify_GivenRulesAndFunctionIsMetric() {
@@ -551,7 +551,7 @@ public void testVerify_GivenRulesAndFunctionIsMetric() {
 
         ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build);
 
-        assertThat(e.getMessage(), equalTo("Invalid detector rule: function metric does not support rules with conditions"));
+        assertThat(e.getMessage(), equalTo("Invalid detector rule: function metric only supports conditions that apply to time"));
     }
 
     public void testVerify_GivenRulesAndFunctionIsRare() {
@@ -562,7 +562,7 @@ public void testVerify_GivenRulesAndFunctionIsRare() {
 
         ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build);
 
-        assertThat(e.getMessage(), equalTo("Invalid detector rule: function rare does not support rules with conditions"));
+        assertThat(e.getMessage(), equalTo("Invalid detector rule: function rare only supports conditions that apply to time"));
     }
 
     public void testVerify_GivenRulesAndFunctionIsFreqRare() {
@@ -574,7 +574,7 @@ public void testVerify_GivenRulesAndFunctionIsFreqRare() {
 
         ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build);
 
-        assertThat(e.getMessage(), equalTo("Invalid detector rule: function freq_rare does not support rules with conditions"));
+        assertThat(e.getMessage(), equalTo("Invalid detector rule: function freq_rare only supports conditions that apply to time"));
     }
 
     public void testVerify_GivenTimeConditionRuleAndFunctionIsLatLong() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleScopeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleScopeTests.java
index 10b9c29aba7ef..126e668dba765 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleScopeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleScopeTests.java
@@ -10,6 +10,8 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
+import java.util.Collections;
+
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -53,6 +55,17 @@ public void testValidate_GivenMultipleValidFields() {
         scope.validate(Sets.newHashSet("foo", "bar", "foobar"));
     }
 
+    public void testValidate_GivenNoAvailableFieldsForScope() {
+        RuleScope scope = RuleScope.builder()
+                .include("foo", "filter1")
+                .build();
+        assertThat(scope.isEmpty(), is(false));
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> scope.validate(Collections.emptySet()));
+        assertThat(e.getMessage(), equalTo("Invalid detector rule: scope field 'foo' is invalid; " +
+                "detector has no available fields for scoping"));
+    }
+
     public void testValidate_GivenMultipleFieldsIncludingInvalid() {
         RuleScope scope = RuleScope.builder()
                 .include("foo", "filter1")
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java
index 3e4e4a84d2f8e..d892eb550a17a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java
@@ -5,125 +5,161 @@
  */
 package org.elasticsearch.xpack.core.rollup;
 
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
-import org.elasticsearch.xpack.core.rollup.job.HistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.HistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
 import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween;
+import static com.carrotsearch.randomizedtesting.generators.RandomPicks.randomFrom;
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLengthBetween;
+import static org.elasticsearch.test.ESTestCase.randomDateTimeZone;
+
 public class ConfigTestHelpers {
 
-    public static RollupJobConfig.Builder getRollupJob(String jobId) {
-        RollupJobConfig.Builder builder = new RollupJobConfig.Builder();
-        builder.setId(jobId);
-        builder.setCron(getCronString());
-        builder.setTimeout(new TimeValue(ESTestCase.randomIntBetween(1,100)));
-        String indexPattern = ESTestCase.randomAlphaOfLengthBetween(1,10);
-        builder.setIndexPattern(indexPattern);
-        builder.setRollupIndex("rollup_" + indexPattern); // to ensure the index pattern != rollup index
-        builder.setGroupConfig(ConfigTestHelpers.getGroupConfig().build());
-        builder.setPageSize(ESTestCase.randomIntBetween(1,10));
-        if (ESTestCase.randomBoolean()) {
-            List<MetricConfig> metrics = IntStream.range(1, ESTestCase.randomIntBetween(1,10))
-                    .mapToObj(n -> ConfigTestHelpers.getMetricConfig().build())
-                    .collect(Collectors.toList());
-
-            builder.setMetricsConfig(metrics);
-        }
-        return builder;
+    private static final String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m"};
+
+    private ConfigTestHelpers() {
     }
 
-    public static GroupConfig.Builder getGroupConfig() {
-        GroupConfig.Builder groupBuilder = new GroupConfig.Builder();
-        groupBuilder.setDateHisto(getDateHisto().build());
-        if (ESTestCase.randomBoolean()) {
-            groupBuilder.setHisto(getHisto().build());
-        }
-        if (ESTestCase.randomBoolean()) {
-            groupBuilder.setTerms(getTerms().build());
+    public static RollupJobConfig randomRollupJobConfig(final Random random) {
+        return randomRollupJobConfig(random, randomAsciiAlphanumOfLengthBetween(random, 5, 20));
+    }
+    public static RollupJobConfig randomRollupJobConfig(final Random random, final String id) {
+        return randomRollupJobConfig(random, id, randomAsciiAlphanumOfLengthBetween(random, 5, 20));
+    }
+
+    public static RollupJobConfig randomRollupJobConfig(final Random random, final String id, final String indexPattern) {
+        return randomRollupJobConfig(random, id, indexPattern, "rollup_" + indexPattern);
+    }
+
+    public static RollupJobConfig randomRollupJobConfig(final Random random,
+                                                        final String id,
+                                                        final String indexPattern,
+                                                        final String rollupIndex) {
+        final String cron = randomCron();
+        final int pageSize = randomIntBetween(random, 1, 10);
+        final TimeValue timeout = random.nextBoolean() ? null : randomTimeout(random);
+        final GroupConfig groups = randomGroupConfig(random);
+        final List<MetricConfig> metrics = random.nextBoolean() ? null : randomMetricsConfigs(random);
+        return new RollupJobConfig(id, indexPattern, rollupIndex, cron, pageSize, groups, metrics, timeout);
+    }
+
+    public static GroupConfig randomGroupConfig(final Random random) {
+        DateHistogramGroupConfig dateHistogram = randomDateHistogramGroupConfig(random);
+        HistogramGroupConfig histogram = random.nextBoolean() ? randomHistogramGroupConfig(random) : null;
+        TermsGroupConfig terms = random.nextBoolean() ? randomTermsGroupConfig(random) : null;
+        return new GroupConfig(dateHistogram, histogram, terms);
+    }
+
+    public static DateHistogramGroupConfig randomDateHistogramGroupConfig(final Random random) {
+        final String field = randomField(random);
+        final DateHistogramInterval interval = randomInterval();
+        final DateHistogramInterval delay = random.nextBoolean() ? randomInterval() : null;
+        final String timezone = random.nextBoolean() ? randomDateTimeZone().toString() : null;
+        return new DateHistogramGroupConfig(field, interval, delay, timezone);
+    }
+
+    public static  List<String> getFields() {
+        return IntStream.range(0, ESTestCase.randomIntBetween(1, 10))
+                .mapToObj(n -> ESTestCase.randomAlphaOfLengthBetween(5, 10))
+                .collect(Collectors.toList());
+    }
+
+    public static String randomCron() {
+        return (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(0, 59)))             + //second
+                " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(0, 59)))      + //minute
+                " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(0, 23)))      + //hour
+                " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(1, 31)))      + //day of month
+                " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(1, 12)))      + //month
+                " ?"                                                                         + //day of week
+                " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(1970, 2199)));  //year
+    }
+
+    public static HistogramGroupConfig randomHistogramGroupConfig(final Random random) {
+        return new HistogramGroupConfig(randomInterval(random), randomFields(random));
+    }
+
+    public static List<MetricConfig> randomMetricsConfigs(final Random random) {
+        final int numMetrics = randomIntBetween(random, 1, 10);
+        final List<MetricConfig> metrics = new ArrayList<>(numMetrics);
+        for (int i = 0; i < numMetrics; i++) {
+            metrics.add(randomMetricConfig(random));
         }
-        return groupBuilder;
+        return Collections.unmodifiableList(metrics);
     }
 
-    public static MetricConfig.Builder getMetricConfig() {
-        MetricConfig.Builder builder = new MetricConfig.Builder();
-        builder.setField(ESTestCase.randomAlphaOfLength(15));  // large names so we don't accidentally collide
-        List<String> metrics = new ArrayList<>();
-        if (ESTestCase.randomBoolean()) {
+    public static MetricConfig randomMetricConfig(final Random random) {
+        final String field = randomAsciiAlphanumOfLengthBetween(random, 15, 25);  // large names so we don't accidentally collide
+        final List<String> metrics = new ArrayList<>();
+        if (random.nextBoolean()) {
             metrics.add("min");
         }
-        if (ESTestCase.randomBoolean()) {
+        if (random.nextBoolean()) {
             metrics.add("max");
         }
-        if (ESTestCase.randomBoolean()) {
+        if (random.nextBoolean()) {
             metrics.add("sum");
         }
-        if (ESTestCase.randomBoolean()) {
+        if (random.nextBoolean()) {
             metrics.add("avg");
         }
-        if (ESTestCase.randomBoolean()) {
+        if (random.nextBoolean()) {
             metrics.add("value_count");
         }
         if (metrics.size() == 0) {
             metrics.add("min");
         }
-        builder.setMetrics(metrics);
-        return builder;
+        return new MetricConfig(field, Collections.unmodifiableList(metrics));
     }
 
-    private static final String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m"};
-    public static String randomPositiveTimeValue() {
-        return ESTestCase.randomIntBetween(1, 1000) + ESTestCase.randomFrom(TIME_SUFFIXES);
+    public static TermsGroupConfig randomTermsGroupConfig(final Random random) {
+        return new TermsGroupConfig(randomFields(random));
     }
 
-    public static DateHistoGroupConfig.Builder getDateHisto() {
-        DateHistoGroupConfig.Builder dateHistoBuilder = new DateHistoGroupConfig.Builder();
-        dateHistoBuilder.setInterval(new DateHistogramInterval(randomPositiveTimeValue()));
-        if (ESTestCase.randomBoolean()) {
-            dateHistoBuilder.setTimeZone(ESTestCase.randomDateTimeZone());
-        }
-        if (ESTestCase.randomBoolean()) {
-            dateHistoBuilder.setDelay(new DateHistogramInterval(randomPositiveTimeValue()));
+    private static String[] randomFields(final Random random) {
+        final int numFields = randomIntBetween(random, 1, 10);
+        final String[] fields = new String[numFields];
+        for (int i = 0; i < numFields; i++) {
+            fields[i] = randomField(random);
         }
-        dateHistoBuilder.setField(ESTestCase.randomAlphaOfLengthBetween(5, 10));
-        return dateHistoBuilder;
+        return fields;
     }
 
-    public static HistoGroupConfig.Builder getHisto() {
-        HistoGroupConfig.Builder histoBuilder = new HistoGroupConfig.Builder();
-        histoBuilder.setInterval(ESTestCase.randomIntBetween(1,10000));
-        histoBuilder.setFields(getFields());
-        return histoBuilder;
+    public static String randomField(final Random random) {
+        return randomAsciiAlphanumOfLengthBetween(random, 5, 10);
     }
 
-    public static TermsGroupConfig.Builder getTerms() {
-        TermsGroupConfig.Builder builder = new TermsGroupConfig.Builder();
-        builder.setFields(getFields());
-        return builder;
+    private static String randomPositiveTimeValue() {
+        return ESTestCase.randomIntBetween(1, 1000) + ESTestCase.randomFrom(TIME_SUFFIXES);
     }
 
-    public static  List<String> getFields() {
-        return IntStream.range(0, ESTestCase.randomIntBetween(1, 10))
-                .mapToObj(n -> ESTestCase.randomAlphaOfLengthBetween(5, 10))
-                .collect(Collectors.toList());
+    public static DateHistogramInterval randomInterval() {
+        return new DateHistogramInterval(randomPositiveTimeValue());
     }
 
-    public static String getCronString() {
-        return (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(0, 59)))             + //second
-                " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(0, 59)))      + //minute
-                " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(0, 23)))      + //hour
-                " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(1, 31)))      + //day of month
-                " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(1, 12)))      + //month
-                " ?"                                                                         + //day of week
-                " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(1970, 2199)));  //year
+    private static long randomInterval(final Random random) {
+        return RandomNumbers.randomLongBetween(random, 1L, Long.MAX_VALUE);
     }
+
+    public static TimeValue randomTimeout(final Random random) {
+        return new TimeValue(randomIntBetween(random, 0, 60),
+            randomFrom(random, Arrays.asList(TimeUnit.MILLISECONDS, TimeUnit.SECONDS, TimeUnit.MINUTES)));
+    }
+
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistoGroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java
similarity index 55%
rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistoGroupConfigSerializingTests.java
rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java
index 034f16a100084..6b8846def7284 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistoGroupConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java
@@ -7,52 +7,52 @@
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
+import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomDateHistogramGroupConfig;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class DateHistoGroupConfigSerializingTests extends AbstractSerializingTestCase<DateHistoGroupConfig> {
+public class DateHistogramGroupConfigSerializingTests extends AbstractSerializingTestCase<DateHistogramGroupConfig> {
     @Override
-    protected DateHistoGroupConfig doParseInstance(XContentParser parser) throws IOException {
-        return DateHistoGroupConfig.PARSER.apply(parser, null).build();
+    protected DateHistogramGroupConfig doParseInstance(final XContentParser parser) throws IOException {
+        return DateHistogramGroupConfig.fromXContent(parser);
     }
 
     @Override
-    protected Writeable.Reader<DateHistoGroupConfig> instanceReader() {
-        return DateHistoGroupConfig::new;
+    protected Writeable.Reader<DateHistogramGroupConfig> instanceReader() {
+        return DateHistogramGroupConfig::new;
     }
 
     @Override
-    protected DateHistoGroupConfig createTestInstance() {
-        return ConfigTestHelpers.getDateHisto().build();
+    protected DateHistogramGroupConfig createTestInstance() {
+        return randomDateHistogramGroupConfig(random());
     }
 
-    public void testValidateNoMapping() throws IOException {
+    public void testValidateNoMapping() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
-        DateHistoGroupConfig config = new DateHistoGroupConfig.Builder()
-                .setField("my_field")
-                .setInterval(new DateHistogramInterval("1d"))
-                .build();
+        DateHistogramGroupConfig config = new DateHistogramGroupConfig("my_field", new DateHistogramInterval("1d"), null, null);
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("Could not find a [date] field with name [my_field] in any of the " +
                 "indices matching the index pattern."));
     }
 
-    public void testValidateNomatchingField() throws IOException {
-
+    public void testValidateNomatchingField() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -60,17 +60,13 @@ public void testValidateNomatchingField() throws IOException {
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         responseMap.put("some_other_field", Collections.singletonMap("date", fieldCaps));
 
-        DateHistoGroupConfig config = new DateHistoGroupConfig.Builder()
-                .setField("my_field")
-                .setInterval(new DateHistogramInterval("1d"))
-                .build();
+        DateHistogramGroupConfig config = new DateHistogramGroupConfig("my_field", new DateHistogramInterval("1d"), null, null);
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("Could not find a [date] field with name [my_field] in any of the " +
                 "indices matching the index pattern."));
     }
 
-    public void testValidateFieldWrongType() throws IOException {
-
+    public void testValidateFieldWrongType() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -78,17 +74,13 @@ public void testValidateFieldWrongType() throws IOException {
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         responseMap.put("my_field", Collections.singletonMap("keyword", fieldCaps));
 
-        DateHistoGroupConfig config = new DateHistoGroupConfig.Builder()
-                .setField("my_field")
-                .setInterval(new DateHistogramInterval("1d"))
-                .build();
+        DateHistogramGroupConfig config = new DateHistogramGroupConfig("my_field", new DateHistogramInterval("1d"), null, null);
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be a [date] type across all " +
                 "indices in the index pattern.  Found: [keyword] for field [my_field]"));
     }
 
-    public void testValidateFieldMixtureTypes() throws IOException {
-
+    public void testValidateFieldMixtureTypes() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -99,17 +91,13 @@ public void testValidateFieldMixtureTypes() throws IOException {
         types.put("keyword", fieldCaps);
         responseMap.put("my_field", types);
 
-        DateHistoGroupConfig config = new DateHistoGroupConfig.Builder()
-                .setField("my_field")
-                .setInterval(new DateHistogramInterval("1d"))
-                .build();
+        DateHistogramGroupConfig config = new DateHistogramGroupConfig("my_field", new DateHistogramInterval("1d"), null, null);
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be a [date] type across all " +
                 "indices in the index pattern.  Found: [date, keyword] for field [my_field]"));
     }
 
-    public void testValidateFieldMatchingNotAggregatable() throws IOException {
-
+    public void testValidateFieldMatchingNotAggregatable() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -118,15 +106,12 @@ public void testValidateFieldMatchingNotAggregatable() throws IOException {
         when(fieldCaps.isAggregatable()).thenReturn(false);
         responseMap.put("my_field", Collections.singletonMap("date", fieldCaps));
 
-        DateHistoGroupConfig config = new DateHistoGroupConfig.Builder()
-                .setField("my_field")
-                .setInterval(new DateHistogramInterval("1d"))
-                .build();
+        DateHistogramGroupConfig config =new DateHistogramGroupConfig("my_field", new DateHistogramInterval("1d"), null, null);
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("The field [my_field] must be aggregatable across all indices, but is not."));
     }
 
-    public void testValidateMatchingField() throws IOException {
+    public void testValidateMatchingField() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -135,11 +120,49 @@ public void testValidateMatchingField() throws IOException {
         when(fieldCaps.isAggregatable()).thenReturn(true);
         responseMap.put("my_field", Collections.singletonMap("date", fieldCaps));
 
-        DateHistoGroupConfig config = new DateHistoGroupConfig.Builder()
-                .setField("my_field")
-                .setInterval(new DateHistogramInterval("1d"))
-                .build();
+        DateHistogramGroupConfig config = new DateHistogramGroupConfig("my_field", new DateHistogramInterval("1d"), null, null);
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().size(), equalTo(0));
     }
+
+    /**
+     * Tests that a DateHistogramGroupConfig can be serialized/deserialized correctly after
+     * the timezone was changed from DateTimeZone to String.
+     */
+    public void testBwcSerialization() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            final DateHistogramGroupConfig reference = ConfigTestHelpers.randomDateHistogramGroupConfig(random());
+
+            final BytesStreamOutput out = new BytesStreamOutput();
+            reference.writeTo(out);
+
+            // previous way to deserialize a DateHistogramGroupConfig
+            final StreamInput in = out.bytes().streamInput();
+            DateHistogramInterval interval = new DateHistogramInterval(in);
+            String field = in.readString();
+            DateHistogramInterval delay = in.readOptionalWriteable(DateHistogramInterval::new);
+            DateTimeZone timeZone = in.readTimeZone();
+
+            assertEqualInstances(reference, new DateHistogramGroupConfig(field, interval, delay, timeZone.getID()));
+        }
+
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            final String field = ConfigTestHelpers.randomField(random());
+            final DateHistogramInterval interval = ConfigTestHelpers.randomInterval();
+            final DateHistogramInterval delay = randomBoolean() ? ConfigTestHelpers.randomInterval() : null;
+            final DateTimeZone timezone = randomDateTimeZone();
+
+            // previous way to serialize a DateHistogramGroupConfig
+            final BytesStreamOutput out = new BytesStreamOutput();
+            interval.writeTo(out);
+            out.writeString(field);
+            out.writeOptionalWriteable(delay);
+            out.writeTimeZone(timezone);
+
+            final StreamInput in = out.bytes().streamInput();
+            DateHistogramGroupConfig deserialized = new DateHistogramGroupConfig(in);
+
+            assertEqualInstances(new DateHistogramGroupConfig(field, interval, delay, timezone.getID()), deserialized);
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/GroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/GroupConfigSerializingTests.java
index c220f10aeab27..49ea206ded767 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/GroupConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/GroupConfigSerializingTests.java
@@ -8,14 +8,16 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 
 import java.io.IOException;
 
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomGroupConfig;
+
 public class GroupConfigSerializingTests extends AbstractSerializingTestCase<GroupConfig> {
+
     @Override
-    protected GroupConfig doParseInstance(XContentParser parser) throws IOException {
-        return GroupConfig.PARSER.apply(parser, null).build();
+    protected GroupConfig doParseInstance(final XContentParser parser) throws IOException {
+        return GroupConfig.fromXContent(parser);
     }
 
     @Override
@@ -25,6 +27,6 @@ protected Writeable.Reader<GroupConfig> instanceReader() {
 
     @Override
     protected GroupConfig createTestInstance() {
-        return ConfigTestHelpers.getGroupConfig().build();
+        return randomGroupConfig(random());
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistogramGroupConfigSerializingTests.java
similarity index 74%
rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfigSerializingTests.java
rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistogramGroupConfigSerializingTests.java
index 92e7d8b9643e6..ef81b235b1fba 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistogramGroupConfigSerializingTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.RollupField;
 
 import java.io.IOException;
@@ -18,34 +17,33 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomHistogramGroupConfig;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class HistoGroupConfigSerializingTests extends AbstractSerializingTestCase<HistoGroupConfig> {
+public class HistogramGroupConfigSerializingTests extends AbstractSerializingTestCase<HistogramGroupConfig> {
+
     @Override
-    protected HistoGroupConfig doParseInstance(XContentParser parser) throws IOException {
-        return HistoGroupConfig.PARSER.apply(parser, null).build();
+    protected HistogramGroupConfig doParseInstance(XContentParser parser) throws IOException {
+        return HistogramGroupConfig.fromXContent(parser);
     }
 
     @Override
-    protected Writeable.Reader<HistoGroupConfig> instanceReader() {
-        return HistoGroupConfig::new;
+    protected Writeable.Reader<HistogramGroupConfig> instanceReader() {
+        return HistogramGroupConfig::new;
     }
 
     @Override
-    protected HistoGroupConfig createTestInstance() {
-        return ConfigTestHelpers.getHisto().build();
+    protected HistogramGroupConfig createTestInstance() {
+        return randomHistogramGroupConfig(random());
     }
 
     public void testValidateNoMapping() throws IOException {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
-        HistoGroupConfig config = new HistoGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .setInterval(123)
-                .build();
+        HistogramGroupConfig config = new HistogramGroupConfig(132, "my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] field with name [my_field] in any of the " +
                 "indices matching the index pattern."));
@@ -60,10 +58,7 @@ public void testValidateNomatchingField() throws IOException {
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         responseMap.put("some_other_field", Collections.singletonMap("long", fieldCaps));
 
-        HistoGroupConfig config = new HistoGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .setInterval(123)
-                .build();
+        HistogramGroupConfig config = new HistogramGroupConfig(132, "my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] field with name [my_field] in any of the " +
                 "indices matching the index pattern."));
@@ -78,10 +73,7 @@ public void testValidateFieldWrongType() throws IOException {
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         responseMap.put("my_field", Collections.singletonMap("keyword", fieldCaps));
 
-        HistoGroupConfig config = new HistoGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .setInterval(123)
-                .build();
+        HistogramGroupConfig config = new HistogramGroupConfig(132, "my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("The field referenced by a histo group must be a [numeric] type, but " +
                 "found [keyword] for field [my_field]"));
@@ -97,10 +89,7 @@ public void testValidateFieldMatchingNotAggregatable() throws IOException {
         when(fieldCaps.isAggregatable()).thenReturn(false);
         responseMap.put("my_field", Collections.singletonMap("long", fieldCaps));
 
-        HistoGroupConfig config = new HistoGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .setInterval(123)
-                .build();
+        HistogramGroupConfig config = new HistogramGroupConfig(132, "my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("The field [my_field] must be aggregatable across all indices, but is not."));
     }
@@ -115,10 +104,7 @@ public void testValidateMatchingField() throws IOException {
         String mappingType = randomFrom(RollupField.NUMERIC_FIELD_MAPPER_TYPES);
         responseMap.put("my_field", Collections.singletonMap(mappingType, fieldCaps));
 
-        HistoGroupConfig config = new HistoGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .setInterval(123)
-                .build();
+        HistogramGroupConfig config = new HistogramGroupConfig(132, "my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().size(), equalTo(0));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
index 5515aeaaf4829..a0df63bc38dde 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
@@ -39,8 +39,8 @@ protected GetRollupJobsAction.JobWrapper createTestInstance() {
             state = IndexerState.ABORTING;
         }
 
-        return new GetRollupJobsAction.JobWrapper(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(),
+        return new GetRollupJobsAction.JobWrapper(ConfigTestHelpers.randomRollupJobConfig(random()),
                 new RollupJobStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()),
-                new RollupJobStatus(state, Collections.emptyMap()));
+                new RollupJobStatus(state, Collections.emptyMap(), randomBoolean()));
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricsConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java
similarity index 69%
rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricsConfigSerializingTests.java
rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java
index 9b330e7165093..a5b8d9afead2e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricsConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java
@@ -17,14 +17,16 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class MetricsConfigSerializingTests extends AbstractSerializingTestCase<MetricConfig> {
+public class MetricConfigSerializingTests extends AbstractSerializingTestCase<MetricConfig> {
+
     @Override
-    protected MetricConfig doParseInstance(XContentParser parser) throws IOException {
-        return MetricConfig.PARSER.apply(parser, null).build();
+    protected MetricConfig doParseInstance(final XContentParser parser) throws IOException {
+        return MetricConfig.fromXContent(parser);
     }
 
     @Override
@@ -34,24 +36,20 @@ protected Writeable.Reader<MetricConfig> instanceReader() {
 
     @Override
     protected MetricConfig createTestInstance() {
-        return ConfigTestHelpers.getMetricConfig().build();
+        return ConfigTestHelpers.randomMetricConfig(random());
     }
 
-    public void testValidateNoMapping() throws IOException {
+    public void testValidateNoMapping() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
-        MetricConfig config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        MetricConfig config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] field with name [my_field] in any of the " +
                 "indices matching the index pattern."));
     }
 
-    public void testValidateNomatchingField() throws IOException {
-
+    public void testValidateNomatchingField() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -59,17 +57,13 @@ public void testValidateNomatchingField() throws IOException {
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         responseMap.put("some_other_field", Collections.singletonMap("date", fieldCaps));
 
-        MetricConfig config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        MetricConfig config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] field with name [my_field] in any of the " +
                 "indices matching the index pattern."));
     }
 
-    public void testValidateFieldWrongType() throws IOException {
-
+    public void testValidateFieldWrongType() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -77,17 +71,13 @@ public void testValidateFieldWrongType() throws IOException {
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         responseMap.put("my_field", Collections.singletonMap("keyword", fieldCaps));
 
-        MetricConfig config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        MetricConfig config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("The field referenced by a metric group must be a [numeric] type, " +
                 "but found [keyword] for field [my_field]"));
     }
 
-    public void testValidateFieldMatchingNotAggregatable() throws IOException {
-
+    public void testValidateFieldMatchingNotAggregatable() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -96,15 +86,12 @@ public void testValidateFieldMatchingNotAggregatable() throws IOException {
         when(fieldCaps.isAggregatable()).thenReturn(false);
         responseMap.put("my_field", Collections.singletonMap("long", fieldCaps));
 
-        MetricConfig config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        MetricConfig config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("The field [my_field] must be aggregatable across all indices, but is not."));
     }
 
-    public void testValidateMatchingField() throws IOException {
+    public void testValidateMatchingField() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -113,10 +100,7 @@ public void testValidateMatchingField() throws IOException {
         when(fieldCaps.isAggregatable()).thenReturn(true);
         responseMap.put("my_field", Collections.singletonMap("long", fieldCaps));
 
-        MetricConfig config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        MetricConfig config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().size(), equalTo(0));
 
@@ -124,70 +108,49 @@ public void testValidateMatchingField() throws IOException {
         fieldCaps = mock(FieldCapabilities.class);
         when(fieldCaps.isAggregatable()).thenReturn(true);
         responseMap.put("my_field", Collections.singletonMap("double", fieldCaps));
-        config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().size(), equalTo(0));
 
         fieldCaps = mock(FieldCapabilities.class);
         when(fieldCaps.isAggregatable()).thenReturn(true);
         responseMap.put("my_field", Collections.singletonMap("float", fieldCaps));
-        config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().size(), equalTo(0));
 
         fieldCaps = mock(FieldCapabilities.class);
         when(fieldCaps.isAggregatable()).thenReturn(true);
         responseMap.put("my_field", Collections.singletonMap("short", fieldCaps));
-        config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().size(), equalTo(0));
 
         fieldCaps = mock(FieldCapabilities.class);
         when(fieldCaps.isAggregatable()).thenReturn(true);
         responseMap.put("my_field", Collections.singletonMap("byte", fieldCaps));
-        config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().size(), equalTo(0));
 
         fieldCaps = mock(FieldCapabilities.class);
         when(fieldCaps.isAggregatable()).thenReturn(true);
         responseMap.put("my_field", Collections.singletonMap("half_float", fieldCaps));
-        config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().size(), equalTo(0));
 
         fieldCaps = mock(FieldCapabilities.class);
         when(fieldCaps.isAggregatable()).thenReturn(true);
         responseMap.put("my_field", Collections.singletonMap("scaled_float", fieldCaps));
-        config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().size(), equalTo(0));
 
         fieldCaps = mock(FieldCapabilities.class);
         when(fieldCaps.isAggregatable()).thenReturn(true);
         responseMap.put("my_field", Collections.singletonMap("integer", fieldCaps));
-        config = new MetricConfig.Builder()
-                .setField("my_field")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().size(), equalTo(0));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java
index a5f03aab51830..a5a82bc2bb090 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java
@@ -8,17 +8,27 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
+import org.junit.Before;
 
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomRollupJobConfig;
 import static org.hamcrest.Matchers.equalTo;
 
 
 public class RollupJobConfigTests extends AbstractSerializingTestCase<RollupJobConfig> {
 
+    private String jobId;
+
+    @Before
+    public void setUpOptionalId() {
+        jobId = randomAlphaOfLengthBetween(1, 10);
+    }
+
     @Override
     protected RollupJobConfig createTestInstance() {
-        return ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1,10)).build();
+        return randomRollupJobConfig(random(), jobId);
     }
 
     @Override
@@ -27,43 +37,139 @@ protected Writeable.Reader<RollupJobConfig> instanceReader() {
     }
 
     @Override
-    protected RollupJobConfig doParseInstance(XContentParser parser) {
-        return RollupJobConfig.PARSER.apply(parser, null).build();
+    protected RollupJobConfig doParseInstance(final XContentParser parser) throws IOException {
+        if (randomBoolean()) {
+            return RollupJobConfig.fromXContent(parser, jobId);
+        } else {
+            return RollupJobConfig.fromXContent(parser, null);
+        }
     }
 
     public void testEmptyIndexPattern() {
-        RollupJobConfig.Builder builder = ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1, 10));
-        builder.setIndexPattern(null);
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build);
-        assertThat(e.getMessage(), equalTo("An index pattern is mandatory."));
-
-        builder = ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1, 10));
-        builder.setIndexPattern("");
-        e = expectThrows(IllegalArgumentException.class, builder::build);
-        assertThat(e.getMessage(), equalTo("An index pattern is mandatory."));
+        final RollupJobConfig sample = randomRollupJobConfig(random());
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), null, sample.getRollupIndex(), sample.getCron(), sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Index pattern must be a non-null, non-empty string"));
+
+        e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), "", sample.getRollupIndex(), sample.getCron(), sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Index pattern must be a non-null, non-empty string"));
     }
 
     public void testEmptyCron() {
-        RollupJobConfig.Builder builder = ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1, 10));
-        builder.setCron(null);
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build);
-        assertThat(e.getMessage(), equalTo("A cron schedule is mandatory."));
-
-        builder = ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1, 10));
-        builder.setCron("");
-        e = expectThrows(IllegalArgumentException.class, builder::build);
-        assertThat(e.getMessage(), equalTo("A cron schedule is mandatory."));
+        final RollupJobConfig sample = randomRollupJobConfig(random());
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), null, sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Cron schedule must be a non-null, non-empty string"));
+
+        e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), "", sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Cron schedule must be a non-null, non-empty string"));
     }
 
     public void testEmptyID() {
-        RollupJobConfig.Builder builder = ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1, 10));
-        builder.setId(null);
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build);
-        assertThat(e.getMessage(), equalTo("An ID is mandatory."));
-
-        builder = ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1, 10));
-        builder.setId("");
-        e = expectThrows(IllegalArgumentException.class, builder::build);
-        assertThat(e.getMessage(), equalTo("An ID is mandatory."));
+        final RollupJobConfig sample = randomRollupJobConfig(random());
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(null, sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Id must be a non-null, non-empty string"));
+
+        e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig("", sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Id must be a non-null, non-empty string"));
+    }
+
+    public void testBadCron() {
+        final RollupJobConfig sample = randomRollupJobConfig(random());
+
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), "0 * * *", sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("invalid cron expression [0 * * *]"));
+    }
+
+    public void testMatchAllIndexPattern() {
+        final RollupJobConfig sample = randomRollupJobConfig(random());
+
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), "*", sample.getRollupIndex(), sample.getCron(), sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Index pattern must not match all indices (as it would match it's own rollup index"));
+    }
+
+    public void testMatchOwnRollupPatternPrefix() {
+        final RollupJobConfig sample = randomRollupJobConfig(random());
+
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), "foo-*", "foo-rollup", sample.getCron(), sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed"));
+    }
+
+    public void testMatchOwnRollupPatternSuffix() {
+        final RollupJobConfig sample = randomRollupJobConfig(random());
+
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), "*-rollup", "foo-rollup", sample.getCron(), sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed"));
+    }
+
+    public void testIndexPatternIdenticalToRollup() {
+        final RollupJobConfig sample = randomRollupJobConfig(random());
+
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), "foo", "foo", sample.getCron(), sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Rollup index may not be the same as the index pattern"));
+    }
+
+    public void testEmptyRollupIndex() {
+        final RollupJobConfig sample = randomRollupJobConfig(random());
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), sample.getIndexPattern(), "", sample.getCron(), sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Rollup index must be a non-null, non-empty string"));
+
+        e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), sample.getIndexPattern(), null, sample.getCron(), sample.getPageSize(),
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Rollup index must be a non-null, non-empty string"));
+    }
+
+    public void testBadSize() {
+        final RollupJobConfig sample = randomRollupJobConfig(random());
+
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), -1,
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Page size is mandatory and  must be a positive long"));
+
+        e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), 0,
+                sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("Page size is mandatory and  must be a positive long"));
+    }
+
+    public void testEmptyGroupAndMetrics() {
+        final RollupJobConfig sample = randomRollupJobConfig(random());
+
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), sample.getPageSize(),
+                null, null, sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("At least one grouping or metric must be configured"));
+
+        e = expectThrows(IllegalArgumentException.class, () ->
+            new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), sample.getPageSize(),
+                null, emptyList(), sample.getTimeout()));
+        assertThat(e.getMessage(), equalTo("At least one grouping or metric must be configured"));
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatusTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatusTests.java
index cc9b9e81cfbe9..2c802a7e41dc3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatusTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatusTests.java
@@ -8,8 +8,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.rollup.job.IndexerState;
-import org.elasticsearch.xpack.core.rollup.job.RollupJobStatus;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -35,7 +33,7 @@ private Map<String, Object> randomPosition() {
 
     @Override
     protected RollupJobStatus createTestInstance() {
-        return new RollupJobStatus(randomFrom(IndexerState.values()), randomPosition());
+        return new RollupJobStatus(randomFrom(IndexerState.values()), randomPosition(), randomBoolean());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java
index 915cfc2fe3575..3d2367f12bf29 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java
@@ -37,7 +37,7 @@ protected Writeable.Reader instanceReader() {
     @Override
     protected Writeable createTestInstance() {
         if (randomBoolean()) {
-            return new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), null);
+            return new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), null);
         }
 
         Map<String, String> headers = Collections.emptyMap();
@@ -45,7 +45,7 @@ protected Writeable createTestInstance() {
             headers = new HashMap<>(1);
             headers.put("foo", "bar");
         }
-        return new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), headers);
+        return new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), headers);
     }
 
     @Override
@@ -60,7 +60,7 @@ protected Diffable makeTestChanges(Diffable testInstance) {
                 return new RollupJob(other.getConfig(), null);
             }
         } else {
-            return new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), other.getHeaders());
+            return new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), other.getHeaders());
         }
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java
index f48f678fac85a..ccdd616df7b51 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java
@@ -11,27 +11,23 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomTermsGroupConfig;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class TermsGroupConfigSerializingTests extends AbstractSerializingTestCase<TermsGroupConfig> {
 
-    private static final List<String> FLOAT_TYPES = Arrays.asList("half_float", "float", "double", "scaled_float");
-    private static final List<String> NATURAL_TYPES = Arrays.asList("byte", "short", "integer", "long");
-
     @Override
     protected TermsGroupConfig doParseInstance(XContentParser parser) throws IOException {
-        return TermsGroupConfig.PARSER.apply(parser, null).build();
+        return TermsGroupConfig.fromXContent(parser);
     }
 
     @Override
@@ -41,23 +37,20 @@ protected Writeable.Reader<TermsGroupConfig> instanceReader() {
 
     @Override
     protected TermsGroupConfig createTestInstance() {
-        return ConfigTestHelpers.getTerms().build();
+        return randomTermsGroupConfig(random());
     }
 
-    public void testValidateNoMapping() throws IOException {
+    public void testValidateNoMapping() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
-        TermsGroupConfig config = new TermsGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .build();
+        TermsGroupConfig config = new TermsGroupConfig("my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [keyword/text] field with name " +
                 "[my_field] in any of the indices matching the index pattern."));
     }
 
-    public void testValidateNomatchingField() throws IOException {
-
+    public void testValidateNomatchingField() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -65,16 +58,13 @@ public void testValidateNomatchingField() throws IOException {
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         responseMap.put("some_other_field", Collections.singletonMap("keyword", fieldCaps));
 
-        TermsGroupConfig config = new TermsGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .build();
+        TermsGroupConfig config = new TermsGroupConfig("my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [keyword/text] field with name " +
                 "[my_field] in any of the indices matching the index pattern."));
     }
 
-    public void testValidateFieldWrongType() throws IOException {
-
+    public void testValidateFieldWrongType() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -82,17 +72,13 @@ public void testValidateFieldWrongType() throws IOException {
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         responseMap.put("my_field", Collections.singletonMap("geo_point", fieldCaps));
 
-        TermsGroupConfig config = new TermsGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .build();
+        TermsGroupConfig config = new TermsGroupConfig("my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("The field referenced by a terms group must be a [numeric] or " +
                 "[keyword/text] type, but found [geo_point] for field [my_field]"));
     }
 
-    public void testValidateFieldMatchingNotAggregatable() throws IOException {
-
-
+    public void testValidateFieldMatchingNotAggregatable() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -101,14 +87,12 @@ public void testValidateFieldMatchingNotAggregatable() throws IOException {
         when(fieldCaps.isAggregatable()).thenReturn(false);
         responseMap.put("my_field", Collections.singletonMap(getRandomType(), fieldCaps));
 
-        TermsGroupConfig config = new TermsGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .build();
+        TermsGroupConfig config = new TermsGroupConfig("my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("The field [my_field] must be aggregatable across all indices, but is not."));
     }
 
-    public void testValidateMatchingField() throws IOException {
+    public void testValidateMatchingField() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
         String type = getRandomType();
@@ -118,9 +102,7 @@ public void testValidateMatchingField() throws IOException {
         when(fieldCaps.isAggregatable()).thenReturn(true);
         responseMap.put("my_field", Collections.singletonMap(type, fieldCaps));
 
-        TermsGroupConfig config = new TermsGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .build();
+        TermsGroupConfig config = new TermsGroupConfig("my_field");
         config.validateMappings(responseMap, e);
         if (e.validationErrors().size() != 0) {
             fail(e.getMessage());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
index 0fb3094fca5a1..38857e2170de4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
@@ -80,6 +80,7 @@ public void cleanDirectory() throws Exception {
         bitsetFilterCache.close();
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32457")
     public void testSearch() throws Exception {
         IndexWriter iw = new IndexWriter(directory, newIndexWriterConfig());
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java
index d02d7e5fe975c..c26968ce54aa9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java
@@ -76,7 +76,7 @@
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.junit.After;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
@@ -442,7 +442,7 @@ public String execute() {
                         return "rendered_text";
                     }
                 };
-        
+
         when(scriptService.compile(any(Script.class), eq(TemplateScript.CONTEXT))).thenReturn(compiledTemplate);
 
         XContentBuilder builder = jsonBuilder();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java
index a5c1bbc98d1ba..9c113d4ff0f94 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java
@@ -50,6 +50,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 public class ManageApplicationPrivilegesTests extends ESTestCase {
 
@@ -140,6 +141,19 @@ public void testRequestPredicate() {
         assertThat(cloudAndSwiftypePredicate, not(predicateMatches(putKibana)));
     }
 
+    public void testSecurityForGetAllApplicationPrivileges() {
+        final GetPrivilegesRequest getAll = new GetPrivilegesRequest();
+        getAll.application(null);
+        getAll.privileges(new String[0]);
+
+        assertThat(getAll.validate(), nullValue());
+
+        final ManageApplicationPrivileges kibanaOnly = new ManageApplicationPrivileges(Sets.newHashSet("kibana-*"));
+        final ManageApplicationPrivileges allApps = new ManageApplicationPrivileges(Sets.newHashSet("*"));
+
+        assertThat(kibanaOnly.getRequestPredicate(), not(predicateMatches(getAll)));
+        assertThat(allApps.getRequestPredicate(), predicateMatches(getAll));
+    }
 
     private ManageApplicationPrivileges clone(ManageApplicationPrivileges original) {
         return new ManageApplicationPrivileges(new LinkedHashSet<>(original.getApplicationNames()));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4TransportTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4TransportTests.java
index 3c4ebee2ac59c..6ce7e2aebdef8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4TransportTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4TransportTests.java
@@ -21,22 +21,37 @@
 
 public class SecurityNetty4TransportTests extends ESTestCase {
 
-    public void testGetTransportProfileConfigurations() {
+    public void testGetSecureTransportProfileConfigurations() {
         final Settings settings = Settings.builder()
             .put("path.home", createTempDir())
             .put("xpack.security.transport.ssl.verification_mode", VerificationMode.CERTIFICATE.name())
             .put("transport.profiles.full.xpack.security.ssl.verification_mode", VerificationMode.FULL.name())
             .put("transport.profiles.cert.xpack.security.ssl.verification_mode", VerificationMode.CERTIFICATE.name())
-            .put("transport.profiles.none.xpack.security.ssl.verification_mode", VerificationMode.NONE.name())
             .build();
         final Environment env = TestEnvironment.newEnvironment(settings);
         SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration defaultConfig = sslService.getSSLConfiguration("xpack.security.transport.ssl");
         final Map<String, SSLConfiguration> profileConfigurations = getTransportProfileConfigurations(settings, sslService, defaultConfig);
-        assertThat(profileConfigurations.size(), Matchers.equalTo(4));
-        assertThat(profileConfigurations.keySet(), Matchers.containsInAnyOrder("full", "cert", "none", "default"));
+        assertThat(profileConfigurations.size(), Matchers.equalTo(3));
+        assertThat(profileConfigurations.keySet(), Matchers.containsInAnyOrder("full", "cert", "default"));
         assertThat(profileConfigurations.get("full").verificationMode(), Matchers.equalTo(VerificationMode.FULL));
         assertThat(profileConfigurations.get("cert").verificationMode(), Matchers.equalTo(VerificationMode.CERTIFICATE));
+        assertThat(profileConfigurations.get("default"), Matchers.sameInstance(defaultConfig));
+    }
+
+    public void testGetInsecureTransportProfileConfigurations() {
+        assumeFalse("Can't run in a FIPS JVM with verification mode None", inFipsJvm());
+        final Settings settings = Settings.builder()
+            .put("path.home", createTempDir())
+            .put("xpack.security.transport.ssl.verification_mode", VerificationMode.CERTIFICATE.name())
+            .put("transport.profiles.none.xpack.security.ssl.verification_mode", VerificationMode.NONE.name())
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        SSLService sslService = new SSLService(settings, env);
+        final SSLConfiguration defaultConfig = sslService.getSSLConfiguration("xpack.security.transport.ssl");
+        final Map<String, SSLConfiguration> profileConfigurations = getTransportProfileConfigurations(settings, sslService, defaultConfig);
+        assertThat(profileConfigurations.size(), Matchers.equalTo(2));
+        assertThat(profileConfigurations.keySet(), Matchers.containsInAnyOrder("none", "default"));
         assertThat(profileConfigurations.get("none").verificationMode(), Matchers.equalTo(VerificationMode.NONE));
         assertThat(profileConfigurations.get("default"), Matchers.sameInstance(defaultConfig));
     }
diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle
index a673016133724..3602e1b359ec2 100644
--- a/x-pack/plugin/ml/build.gradle
+++ b/x-pack/plugin/ml/build.gradle
@@ -47,7 +47,7 @@ dependencies {
 
     // ml deps
     compile project(':libs:grok')
-    compile 'net.sf.supercsv:super-csv:2.4.0'
+    compile "net.sf.supercsv:super-csv:${versions.supercsv}"
     nativeBundle "org.elasticsearch.ml:ml-cpp:${project.version}@zip"
     testCompile 'org.ini4j:ini4j:0.5.2'
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index d5461d855998b..f2fdea45ebe24 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -162,12 +162,12 @@
 import org.elasticsearch.xpack.ml.job.categorization.MlClassicTokenizer;
 import org.elasticsearch.xpack.ml.job.categorization.MlClassicTokenizerFactory;
 import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.process.DataCountsReporter;
 import org.elasticsearch.xpack.ml.job.process.NativeController;
 import org.elasticsearch.xpack.ml.job.process.NativeControllerHolder;
-import org.elasticsearch.xpack.ml.job.process.ProcessCtrl;
+import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectBuilder;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessFactory;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
 import org.elasticsearch.xpack.ml.job.process.autodetect.BlackHoleAutodetectProcess;
@@ -284,8 +284,8 @@ public List<Setting<?>> getSettings() {
                         CONCURRENT_JOB_ALLOCATIONS,
                         MachineLearningField.MAX_MODEL_MEMORY_LIMIT,
                         MAX_MACHINE_MEMORY_PERCENT,
-                        ProcessCtrl.DONT_PERSIST_MODEL_STATE_SETTING,
-                        ProcessCtrl.MAX_ANOMALY_RECORDS_SETTING,
+                        AutodetectBuilder.DONT_PERSIST_MODEL_STATE_SETTING,
+                        AutodetectBuilder.MAX_ANOMALY_RECORDS_SETTING,
                         DataCountsReporter.ACCEPTABLE_PERCENTAGE_DATE_PARSE_ERRORS_SETTING,
                         DataCountsReporter.ACCEPTABLE_PERCENTAGE_OUT_OF_ORDER_ERRORS_SETTING,
                         AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE,
@@ -357,9 +357,9 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         }
 
         Auditor auditor = new Auditor(client, clusterService.nodeName());
-        JobProvider jobProvider = new JobProvider(client, settings);
+        JobResultsProvider jobResultsProvider = new JobResultsProvider(client, settings);
         UpdateJobProcessNotifier notifier = new UpdateJobProcessNotifier(settings, client, clusterService, threadPool);
-        JobManager jobManager = new JobManager(env, settings, jobProvider, clusterService, auditor, client, notifier);
+        JobManager jobManager = new JobManager(env, settings, jobResultsProvider, clusterService, auditor, client, notifier);
 
         JobDataCountsPersister jobDataCountsPersister = new JobDataCountsPersister(settings, client);
         JobResultsPersister jobResultsPersister = new JobResultsPersister(settings, client);
@@ -390,10 +390,10 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         NormalizerFactory normalizerFactory = new NormalizerFactory(normalizerProcessFactory,
                 threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME));
         AutodetectProcessManager autodetectProcessManager = new AutodetectProcessManager(env, settings, client, threadPool,
-                jobManager, jobProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory,
+                jobManager, jobResultsProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory,
                 normalizerFactory, xContentRegistry, auditor);
         this.autodetectProcessManager.set(autodetectProcessManager);
-        DatafeedJobBuilder datafeedJobBuilder = new DatafeedJobBuilder(client, jobProvider, auditor, System::currentTimeMillis);
+        DatafeedJobBuilder datafeedJobBuilder = new DatafeedJobBuilder(client, jobResultsProvider, auditor, System::currentTimeMillis);
         DatafeedManager datafeedManager = new DatafeedManager(threadPool, client, clusterService, datafeedJobBuilder,
                 System::currentTimeMillis, auditor);
         this.datafeedManager.set(datafeedManager);
@@ -408,7 +408,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
 
         return Arrays.asList(
                 mlLifeCycleService,
-                jobProvider,
+                jobResultsProvider,
                 jobManager,
                 autodetectProcessManager,
                 new MlInitializationService(settings, threadPool, clusterService, client),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java
index 9c712efe693ca..c171acd71b59b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.xpack.core.ml.action.DeleteCalendarAction;
 import org.elasticsearch.xpack.core.ml.calendars.Calendar;
 import org.elasticsearch.xpack.ml.job.JobManager;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.util.function.Supplier;
 
@@ -35,16 +35,17 @@ public class TransportDeleteCalendarAction extends HandledTransportAction<Delete
 
     private final Client client;
     private final JobManager jobManager;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
 
     @Inject
     public TransportDeleteCalendarAction(Settings settings, TransportService transportService,
-                                         ActionFilters actionFilters, Client client, JobManager jobManager, JobProvider jobProvider) {
+                                         ActionFilters actionFilters, Client client, JobManager jobManager,
+                                         JobResultsProvider jobResultsProvider) {
         super(settings, DeleteCalendarAction.NAME, transportService, actionFilters,
             (Supplier<DeleteCalendarAction.Request>) DeleteCalendarAction.Request::new);
         this.client = client;
         this.jobManager = jobManager;
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
     }
 
     @Override
@@ -70,7 +71,7 @@ protected void doExecute(Task task, DeleteCalendarAction.Request request, Action
                 listener::onFailure
         );
 
-        jobProvider.calendar(calendarId, calendarListener);
+        jobResultsProvider.calendar(calendarId, calendarListener);
     }
 
     private DeleteByQueryRequest buildDeleteByQuery(String calendarId) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java
index 52896751de1d3..c52f144923e08 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.xpack.core.ml.calendars.Calendar;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.job.JobManager;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.util.Map;
 
@@ -37,16 +37,16 @@ public class TransportDeleteCalendarEventAction extends HandledTransportAction<D
         DeleteCalendarEventAction.Response> {
 
     private final Client client;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final JobManager jobManager;
 
     @Inject
     public TransportDeleteCalendarEventAction(Settings settings, TransportService transportService, ActionFilters actionFilters,
-                                              Client client, JobProvider jobProvider, JobManager jobManager) {
+                                              Client client, JobResultsProvider jobResultsProvider, JobManager jobManager) {
         super(settings, DeleteCalendarEventAction.NAME, transportService, actionFilters,
               DeleteCalendarEventAction.Request::new);
         this.client = client;
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.jobManager = jobManager;
     }
 
@@ -87,7 +87,7 @@ protected void doExecute(Task task, DeleteCalendarEventAction.Request request,
                 }, listener::onFailure);
 
         // Get the calendar first so we check the calendar exists before checking the event exists
-        jobProvider.calendar(request.getCalendarId(), calendarListener);
+        jobResultsProvider.calendar(request.getCalendarId(), calendarListener);
     }
 
     private void deleteEvent(String eventId, Calendar calendar, ActionListener<DeleteCalendarEventAction.Response> listener) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java
index c63f8a4405b89..398b9930f7d45 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.xpack.core.ml.job.persistence.JobDataDeleter;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.ml.job.JobManager;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
 
 import java.util.Collections;
@@ -32,17 +32,18 @@ public class TransportDeleteModelSnapshotAction extends HandledTransportAction<D
         DeleteModelSnapshotAction.Response> {
 
     private final Client client;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final ClusterService clusterService;
     private final Auditor auditor;
 
     @Inject
     public TransportDeleteModelSnapshotAction(Settings settings, TransportService transportService, ActionFilters actionFilters,
-                                              JobProvider jobProvider, ClusterService clusterService, Client client, Auditor auditor) {
+                                              JobResultsProvider jobResultsProvider, ClusterService clusterService, Client client,
+                                              Auditor auditor) {
         super(settings, DeleteModelSnapshotAction.NAME, transportService, actionFilters,
               DeleteModelSnapshotAction.Request::new);
         this.client = client;
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.clusterService = clusterService;
         this.auditor = auditor;
     }
@@ -51,7 +52,7 @@ public TransportDeleteModelSnapshotAction(Settings settings, TransportService tr
     protected void doExecute(Task task, DeleteModelSnapshotAction.Request request,
                              ActionListener<DeleteModelSnapshotAction.Response> listener) {
         // Verify the snapshot exists
-        jobProvider.modelSnapshots(
+        jobResultsProvider.modelSnapshots(
                 request.getJobId(), 0, 1, null, null, null, true, request.getSnapshotId(),
                 page -> {
                     List<ModelSnapshot> deleteCandidates = page.results();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java
index f42f7003b909c..ad9e6a7c2630a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.job.JobManager;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.ForecastParams;
 
@@ -40,15 +40,15 @@ public class TransportForecastJobAction extends TransportJobTaskAction<ForecastJ
 
     private static final ByteSizeValue FORECAST_LOCAL_STORAGE_LIMIT = new ByteSizeValue(500, ByteSizeUnit.MB);
 
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     @Inject
     public TransportForecastJobAction(Settings settings, TransportService transportService,
                                       ClusterService clusterService, ActionFilters actionFilters,
-                                      JobProvider jobProvider, AutodetectProcessManager processManager) {
+                                      JobResultsProvider jobResultsProvider, AutodetectProcessManager processManager) {
         super(settings, ForecastJobAction.NAME, clusterService, transportService, actionFilters,
             ForecastJobAction.Request::new, ForecastJobAction.Response::new,
                 ThreadPool.Names.SAME, processManager);
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         // ThreadPool.Names.SAME, because operations is executed by autodetect worker thread
     }
 
@@ -107,7 +107,7 @@ protected void taskOperation(ForecastJobAction.Request request, TransportOpenJob
                     }
                 };
 
-                jobProvider.getForecastRequestStats(request.getJobId(), params.getForecastId(),
+                jobResultsProvider.getForecastRequestStats(request.getJobId(), params.getForecastId(),
                         forecastRequestStatsHandler, listener::onFailure);
             } else {
                 listener.onFailure(e);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java
index 63a1efe471a47..377c5a6239289 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java
@@ -16,22 +16,23 @@
 import org.elasticsearch.xpack.core.ml.action.GetBucketsAction;
 import org.elasticsearch.xpack.ml.job.JobManager;
 import org.elasticsearch.xpack.ml.job.persistence.BucketsQueryBuilder;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.util.function.Supplier;
 
 public class TransportGetBucketsAction extends HandledTransportAction<GetBucketsAction.Request, GetBucketsAction.Response> {
 
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final JobManager jobManager;
     private final Client client;
 
     @Inject
     public TransportGetBucketsAction(Settings settings, TransportService transportService,
-                                     ActionFilters actionFilters, JobProvider jobProvider, JobManager jobManager, Client client) {
+                                     ActionFilters actionFilters, JobResultsProvider jobResultsProvider,
+                                     JobManager jobManager, Client client) {
         super(settings, GetBucketsAction.NAME, transportService, actionFilters,
             (Supplier<GetBucketsAction.Request>) GetBucketsAction.Request::new);
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.jobManager = jobManager;
         this.client = client;
     }
@@ -59,7 +60,7 @@ protected void doExecute(Task task, GetBucketsAction.Request request, ActionList
             query.start(request.getStart());
             query.end(request.getEnd());
         }
-        jobProvider.buckets(request.getJobId(), query, q ->
+        jobResultsProvider.buckets(request.getJobId(), query, q ->
                 listener.onResponse(new GetBucketsAction.Response(q)), listener::onFailure, client);
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java
index 2e30ad80d859a..96ba9e6fbbebf 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.ScheduledEventsQueryBuilder;
 
 import java.util.Collections;
@@ -31,15 +31,16 @@
 public class TransportGetCalendarEventsAction extends HandledTransportAction<GetCalendarEventsAction.Request,
         GetCalendarEventsAction.Response> {
 
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final ClusterService clusterService;
 
     @Inject
     public TransportGetCalendarEventsAction(Settings settings, TransportService transportService,
-                                            ActionFilters actionFilters, ClusterService clusterService, JobProvider jobProvider) {
+                                            ActionFilters actionFilters, ClusterService clusterService,
+                                            JobResultsProvider jobResultsProvider) {
         super(settings, GetCalendarEventsAction.NAME, transportService, actionFilters,
             (Supplier<GetCalendarEventsAction.Request>) GetCalendarEventsAction.Request::new);
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.clusterService = clusterService;
     }
 
@@ -85,9 +86,9 @@ protected void doExecute(Task task, GetCalendarEventsAction.Request request,
                             jobGroups = job.getGroups();
                         }
 
-                        jobProvider.scheduledEventsForJob(requestId, jobGroups, query, eventsListener);
+                        jobResultsProvider.scheduledEventsForJob(requestId, jobGroups, query, eventsListener);
                     } else {
-                        jobProvider.scheduledEvents(query, eventsListener);
+                        jobResultsProvider.scheduledEvents(query, eventsListener);
                     }
                 },
                 listener::onFailure);
@@ -101,7 +102,7 @@ private void checkCalendarExists(String calendarId, ActionListener<Boolean> list
             return;
         }
 
-        jobProvider.calendar(calendarId, ActionListener.wrap(
+        jobResultsProvider.calendar(calendarId, ActionListener.wrap(
                 c -> listener.onResponse(true),
                 listener::onFailure
         ));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarsAction.java
index ed837139ade1c..e9a9cd06d92c5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarsAction.java
@@ -17,20 +17,20 @@
 import org.elasticsearch.xpack.core.ml.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.calendars.Calendar;
 import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.util.Collections;
 
 public class TransportGetCalendarsAction extends HandledTransportAction<GetCalendarsAction.Request, GetCalendarsAction.Response> {
 
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
 
     @Inject
     public TransportGetCalendarsAction(Settings settings, TransportService transportService,
-                                       ActionFilters actionFilters, JobProvider jobProvider) {
+                                       ActionFilters actionFilters, JobResultsProvider jobResultsProvider) {
         super(settings, GetCalendarsAction.NAME, transportService, actionFilters,
             GetCalendarsAction.Request::new);
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
     }
 
     @Override
@@ -49,7 +49,7 @@ protected void doExecute(Task task, GetCalendarsAction.Request request, ActionLi
 
     private void getCalendar(String calendarId, ActionListener<GetCalendarsAction.Response> listener) {
 
-        jobProvider.calendar(calendarId, ActionListener.wrap(
+        jobResultsProvider.calendar(calendarId, ActionListener.wrap(
                 calendar -> {
                     QueryPage<Calendar> page = new QueryPage<>(Collections.singletonList(calendar), 1, Calendar.RESULTS_FIELD);
                     listener.onResponse(new GetCalendarsAction.Response(page));
@@ -60,7 +60,7 @@ private void getCalendar(String calendarId, ActionListener<GetCalendarsAction.Re
 
     private void getCalendars(PageParams pageParams, ActionListener<GetCalendarsAction.Response> listener) {
         CalendarQueryBuilder query = new CalendarQueryBuilder().pageParams(pageParams).sort(true);
-        jobProvider.calendars(query, ActionListener.wrap(
+        jobResultsProvider.calendars(query, ActionListener.wrap(
                 calendars -> {
                     listener.onResponse(new GetCalendarsAction.Response(calendars));
                 },
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java
index 0e0481f394ccf..4bac6321a3ebf 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java
@@ -15,22 +15,23 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.GetCategoriesAction;
 import org.elasticsearch.xpack.ml.job.JobManager;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.util.function.Supplier;
 
 public class TransportGetCategoriesAction extends HandledTransportAction<GetCategoriesAction.Request, GetCategoriesAction.Response> {
 
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final Client client;
     private final JobManager jobManager;
 
     @Inject
     public TransportGetCategoriesAction(Settings settings, TransportService transportService,
-                                        ActionFilters actionFilters, JobProvider jobProvider, Client client, JobManager jobManager) {
+                                        ActionFilters actionFilters, JobResultsProvider jobResultsProvider,
+                                        Client client, JobManager jobManager) {
         super(settings, GetCategoriesAction.NAME, transportService, actionFilters,
             (Supplier<GetCategoriesAction.Request>) GetCategoriesAction.Request::new);
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.client = client;
         this.jobManager = jobManager;
     }
@@ -41,7 +42,7 @@ protected void doExecute(Task task, GetCategoriesAction.Request request, ActionL
 
         Integer from = request.getPageParams() != null ? request.getPageParams().getFrom() : null;
         Integer size = request.getPageParams() != null ? request.getPageParams().getSize() : null;
-        jobProvider.categoryDefinitions(request.getJobId(), request.getCategoryId(), true, from, size,
+        jobResultsProvider.categoryDefinitions(request.getJobId(), request.getCategoryId(), true, from, size,
                 r -> listener.onResponse(new GetCategoriesAction.Response(r)), listener::onFailure, client);
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java
index 125e31fcf63cf..e62538831b5cb 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java
@@ -16,22 +16,23 @@
 import org.elasticsearch.xpack.core.ml.action.GetInfluencersAction;
 import org.elasticsearch.xpack.ml.job.JobManager;
 import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.util.function.Supplier;
 
 public class TransportGetInfluencersAction extends HandledTransportAction<GetInfluencersAction.Request, GetInfluencersAction.Response> {
 
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final Client client;
     private final JobManager jobManager;
 
     @Inject
     public TransportGetInfluencersAction(Settings settings, TransportService transportService,
-                                         ActionFilters actionFilters, JobProvider jobProvider, Client client, JobManager jobManager) {
+                                         ActionFilters actionFilters, JobResultsProvider jobResultsProvider,
+                                         Client client, JobManager jobManager) {
         super(settings, GetInfluencersAction.NAME, transportService, actionFilters,
             (Supplier<GetInfluencersAction.Request>) GetInfluencersAction.Request::new);
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.client = client;
         this.jobManager = jobManager;
     }
@@ -49,7 +50,7 @@ protected void doExecute(Task task, GetInfluencersAction.Request request, Action
                 .influencerScoreThreshold(request.getInfluencerScore())
                 .sortField(request.getSort())
                 .sortDescending(request.isDescending()).build();
-        jobProvider.influencers(request.getJobId(), query,
+        jobResultsProvider.influencers(request.getJobId(), query,
                 page -> listener.onResponse(new GetInfluencersAction.Response(page)), listener::onFailure, client);
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
index 28034d757dac2..ab1ef73780e5e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.stats.ForecastStats;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
 
 import java.io.IOException;
@@ -52,18 +52,18 @@ public class TransportGetJobsStatsAction extends TransportTasksAction<TransportO
 
     private final ClusterService clusterService;
     private final AutodetectProcessManager processManager;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
 
     @Inject
     public TransportGetJobsStatsAction(Settings settings, TransportService transportService,
                                        ActionFilters actionFilters, ClusterService clusterService,
-                                       AutodetectProcessManager processManager, JobProvider jobProvider) {
+                                       AutodetectProcessManager processManager, JobResultsProvider jobResultsProvider) {
         super(settings, GetJobsStatsAction.NAME, clusterService, transportService, actionFilters,
             GetJobsStatsAction.Request::new, GetJobsStatsAction.Response::new,
                 ThreadPool.Names.MANAGEMENT);
         this.clusterService = clusterService;
         this.processManager = processManager;
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
     }
 
     @Override
@@ -158,13 +158,13 @@ void gatherStatsForClosedJobs(MlMetadata mlMetadata, GetJobsStatsAction.Request
     }
 
     void gatherForecastStats(String jobId, Consumer<ForecastStats> handler, Consumer<Exception> errorHandler) {
-        jobProvider.getForecastStats(jobId, handler, errorHandler);
+        jobResultsProvider.getForecastStats(jobId, handler, errorHandler);
     }
     
     void gatherDataCountsAndModelSizeStats(String jobId, BiConsumer<DataCounts, ModelSizeStats> handler,
                                                    Consumer<Exception> errorHandler) {
-        jobProvider.dataCounts(jobId, dataCounts -> {
-            jobProvider.modelSizeStats(jobId, modelSizeStats -> {
+        jobResultsProvider.dataCounts(jobId, dataCounts -> {
+            jobResultsProvider.modelSizeStats(jobId, modelSizeStats -> {
                 handler.accept(dataCounts, modelSizeStats);
             }, errorHandler);
         }, errorHandler);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java
index b69db8d48d60f..7a0e0b1c4deb1 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java
@@ -16,22 +16,22 @@
 import org.elasticsearch.xpack.core.ml.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.ml.job.JobManager;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.util.stream.Collectors;
 
 public class TransportGetModelSnapshotsAction extends HandledTransportAction<GetModelSnapshotsAction.Request,
         GetModelSnapshotsAction.Response> {
 
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final JobManager jobManager;
 
     @Inject
     public TransportGetModelSnapshotsAction(Settings settings, TransportService transportService,
-                                            ActionFilters actionFilters, JobProvider jobProvider, JobManager jobManager) {
+                                            ActionFilters actionFilters, JobResultsProvider jobResultsProvider, JobManager jobManager) {
         super(settings, GetModelSnapshotsAction.NAME, transportService, actionFilters,
             GetModelSnapshotsAction.Request::new);
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.jobManager = jobManager;
     }
 
@@ -45,7 +45,7 @@ protected void doExecute(Task task, GetModelSnapshotsAction.Request request,
 
         jobManager.getJobOrThrowIfUnknown(request.getJobId());
 
-        jobProvider.modelSnapshots(request.getJobId(), request.getPageParams().getFrom(), request.getPageParams().getSize(),
+        jobResultsProvider.modelSnapshots(request.getJobId(), request.getPageParams().getFrom(), request.getPageParams().getSize(),
                 request.getStart(), request.getEnd(), request.getSort(), request.getDescOrder(), request.getSnapshotId(),
                 page -> {
                     listener.onResponse(new GetModelSnapshotsAction.Response(clearQuantiles(page)));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java
index b1556ba6e45c7..15a78efd9fda2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java
@@ -15,23 +15,24 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.GetRecordsAction;
 import org.elasticsearch.xpack.ml.job.JobManager;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.RecordsQueryBuilder;
 
 import java.util.function.Supplier;
 
 public class TransportGetRecordsAction extends HandledTransportAction<GetRecordsAction.Request, GetRecordsAction.Response> {
 
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final JobManager jobManager;
     private final Client client;
 
     @Inject
     public TransportGetRecordsAction(Settings settings, TransportService transportService,
-                                     ActionFilters actionFilters, JobProvider jobProvider, JobManager jobManager, Client client) {
+                                     ActionFilters actionFilters, JobResultsProvider jobResultsProvider,
+                                     JobManager jobManager, Client client) {
         super(settings, GetRecordsAction.NAME, transportService, actionFilters,
             (Supplier<GetRecordsAction.Request>) GetRecordsAction.Request::new);
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.jobManager = jobManager;
         this.client = client;
     }
@@ -50,7 +51,7 @@ protected void doExecute(Task task, GetRecordsAction.Request request, ActionList
                 .recordScore(request.getRecordScoreFilter())
                 .sortField(request.getSort())
                 .sortDescending(request.isDescending());
-        jobProvider.records(request.getJobId(), query, page ->
+        jobResultsProvider.records(request.getJobId(), query, page ->
                         listener.onResponse(new GetRecordsAction.Response(page)), listener::onFailure, client);
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
index d378b19aad57a..c4e5793b45171 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
@@ -18,12 +18,14 @@
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -64,7 +66,7 @@
 import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.MachineLearning;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
 
 import java.io.IOException;
@@ -95,19 +97,20 @@ public class TransportOpenJobAction extends TransportMasterNodeAction<OpenJobAct
     private final XPackLicenseState licenseState;
     private final PersistentTasksService persistentTasksService;
     private final Client client;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
 
     @Inject
     public TransportOpenJobAction(Settings settings, TransportService transportService, ThreadPool threadPool,
                                   XPackLicenseState licenseState, ClusterService clusterService,
                                   PersistentTasksService persistentTasksService, ActionFilters actionFilters,
-                                  IndexNameExpressionResolver indexNameExpressionResolver, Client client, JobProvider jobProvider) {
+                                  IndexNameExpressionResolver indexNameExpressionResolver, Client client,
+                                  JobResultsProvider jobResultsProvider) {
         super(settings, OpenJobAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
                 OpenJobAction.Request::new);
         this.licenseState = licenseState;
         this.persistentTasksService = persistentTasksService;
         this.client = client;
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
     }
 
     /**
@@ -470,12 +473,25 @@ protected ClusterBlockException checkBlock(OpenJobAction.Request request, Cluste
     protected void masterOperation(OpenJobAction.Request request, ClusterState state, ActionListener<OpenJobAction.Response> listener) {
         OpenJobAction.JobParams jobParams = request.getJobParams();
         if (licenseState.isMachineLearningAllowed()) {
-            // Step 5. Wait for job to be started and respond
-            ActionListener<PersistentTasksCustomMetaData.PersistentTask<OpenJobAction.JobParams>> finalListener =
+
+            // Step 6. Clear job finished time once the job is started and respond
+            ActionListener<OpenJobAction.Response> clearJobFinishTime = ActionListener.wrap(
+                response -> {
+                    if (response.isAcknowledged()) {
+                        clearJobFinishedTime(jobParams.getJobId(), listener);
+                    } else {
+                        listener.onResponse(response);
+                    }
+                },
+                listener::onFailure
+            );
+
+            // Step 5. Wait for job to be started
+            ActionListener<PersistentTasksCustomMetaData.PersistentTask<OpenJobAction.JobParams>> waitForJobToStart =
                     new ActionListener<PersistentTasksCustomMetaData.PersistentTask<OpenJobAction.JobParams>>() {
                 @Override
                 public void onResponse(PersistentTasksCustomMetaData.PersistentTask<OpenJobAction.JobParams> task) {
-                    waitForJobStarted(task.getId(), jobParams, listener);
+                    waitForJobStarted(task.getId(), jobParams, clearJobFinishTime);
                 }
 
                 @Override
@@ -491,7 +507,7 @@ public void onFailure(Exception e) {
             // Step 4. Start job task
             ActionListener<PutJobAction.Response> establishedMemoryUpdateListener = ActionListener.wrap(
                     response -> persistentTasksService.sendStartRequest(MlTasks.jobTaskId(jobParams.getJobId()),
-                            OpenJobAction.TASK_NAME, jobParams, finalListener),
+                            OpenJobAction.TASK_NAME, jobParams, waitForJobToStart),
                     listener::onFailure
             );
 
@@ -504,7 +520,7 @@ public void onFailure(Exception e) {
                             Long jobEstablishedModelMemory = job.getEstablishedModelMemory();
                             if ((jobVersion == null || jobVersion.before(Version.V_6_1_0))
                                     && (jobEstablishedModelMemory == null || jobEstablishedModelMemory == 0)) {
-                                jobProvider.getEstablishedMemoryUsage(job.getId(), null, null, establishedModelMemory -> {
+                                jobResultsProvider.getEstablishedMemoryUsage(job.getId(), null, null, establishedModelMemory -> {
                                     if (establishedModelMemory != null && establishedModelMemory > 0) {
                                         JobUpdate update = new JobUpdate.Builder(job.getId())
                                                 .setEstablishedModelMemory(establishedModelMemory).build();
@@ -573,6 +589,35 @@ public void onTimeout(TimeValue timeout) {
         });
     }
 
+    private void clearJobFinishedTime(String jobId, ActionListener<OpenJobAction.Response> listener) {
+        clusterService.submitStateUpdateTask("clearing-job-finish-time-for-" + jobId, new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                MlMetadata mlMetadata = MlMetadata.getMlMetadata(currentState);
+                MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(mlMetadata);
+                Job.Builder jobBuilder = new Job.Builder(mlMetadata.getJobs().get(jobId));
+                jobBuilder.setFinishedTime(null);
+
+                mlMetadataBuilder.putJob(jobBuilder.build(), true);
+                ClusterState.Builder builder = ClusterState.builder(currentState);
+                return builder.metaData(new MetaData.Builder(currentState.metaData())
+                    .putCustom(MlMetadata.TYPE, mlMetadataBuilder.build()))
+                    .build();
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                logger.error("[" + jobId + "] Failed to clear finished_time; source [" + source + "]", e);
+                listener.onResponse(new OpenJobAction.Response(true));
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState,
+                                              ClusterState newState) {
+                listener.onResponse(new OpenJobAction.Response(true));
+            }
+        });
+    }
     private void cancelJobStart(PersistentTasksCustomMetaData.PersistentTask<OpenJobAction.JobParams> persistentTask, Exception exception,
                                 ActionListener<OpenJobAction.Response> listener) {
         persistentTasksService.sendRemoveRequest(persistentTask.getId(),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java
index c1279248908a2..7284a490eaa8f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.job.JobManager;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -40,16 +40,17 @@ public class TransportPostCalendarEventsAction extends HandledTransportAction<Po
         PostCalendarEventsAction.Response> {
 
     private final Client client;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final JobManager jobManager;
 
     @Inject
     public TransportPostCalendarEventsAction(Settings settings, TransportService transportService,
-                                             ActionFilters actionFilters, Client client, JobProvider jobProvider, JobManager jobManager) {
+                                             ActionFilters actionFilters, Client client,
+                                             JobResultsProvider jobResultsProvider, JobManager jobManager) {
         super(settings, PostCalendarEventsAction.NAME, transportService, actionFilters,
             PostCalendarEventsAction.Request::new);
         this.client = client;
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.jobManager = jobManager;
     }
 
@@ -92,6 +93,6 @@ public void onFailure(Exception e) {
                 },
                 listener::onFailure);
 
-        jobProvider.calendar(request.getCalendarId(), calendarListener);
+        jobResultsProvider.calendar(request.getCalendarId(), calendarListener);
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
index 68125b8f5cc3c..07b9dade4d8fa 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
@@ -17,8 +17,10 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
@@ -28,7 +30,7 @@
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.job.JobManager;
 import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.util.Date;
 import java.util.function.Consumer;
@@ -38,19 +40,19 @@ public class TransportRevertModelSnapshotAction extends TransportMasterNodeActio
 
     private final Client client;
     private final JobManager jobManager;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final JobDataCountsPersister jobDataCountsPersister;
 
     @Inject
     public TransportRevertModelSnapshotAction(Settings settings, ThreadPool threadPool, TransportService transportService,
                                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                                              JobManager jobManager, JobProvider jobProvider,
+                                              JobManager jobManager, JobResultsProvider jobResultsProvider,
                                               ClusterService clusterService, Client client, JobDataCountsPersister jobDataCountsPersister) {
         super(settings, RevertModelSnapshotAction.NAME, transportService, clusterService, threadPool, actionFilters,
                 indexNameExpressionResolver, RevertModelSnapshotAction.Request::new);
         this.client = client;
         this.jobManager = jobManager;
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.jobDataCountsPersister = jobDataCountsPersister;
     }
 
@@ -66,17 +68,19 @@ protected RevertModelSnapshotAction.Response newResponse() {
 
     @Override
     protected void masterOperation(RevertModelSnapshotAction.Request request, ClusterState state,
-                                   ActionListener<RevertModelSnapshotAction.Response> listener) throws Exception {
+                                   ActionListener<RevertModelSnapshotAction.Response> listener) {
         logger.debug("Received request to revert to snapshot id '{}' for job '{}', deleting intervening results: {}",
                 request.getSnapshotId(), request.getJobId(), request.getDeleteInterveningResults());
 
-        Job job = JobManager.getJobOrThrowIfUnknown(request.getJobId(), clusterService.state());
-        JobState jobState = jobManager.getJobState(job.getId());
+        Job job = JobManager.getJobOrThrowIfUnknown(request.getJobId(), state);
+        PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
+        JobState jobState = MlTasks.getJobState(job.getId(), tasks);
+
         if (jobState.equals(JobState.CLOSED) == false) {
             throw ExceptionsHelper.conflictStatusException(Messages.getMessage(Messages.REST_JOB_NOT_CLOSED_REVERT));
         }
 
-        getModelSnapshot(request, jobProvider, modelSnapshot -> {
+        getModelSnapshot(request, jobResultsProvider, modelSnapshot -> {
             ActionListener<RevertModelSnapshotAction.Response> wrappedListener = listener;
             if (request.getDeleteInterveningResults()) {
                 wrappedListener = wrapDeleteOldDataListener(wrappedListener, modelSnapshot, request.getJobId());
@@ -86,7 +90,7 @@ protected void masterOperation(RevertModelSnapshotAction.Request request, Cluste
         }, listener::onFailure);
     }
 
-    private void getModelSnapshot(RevertModelSnapshotAction.Request request, JobProvider provider, Consumer<ModelSnapshot> handler,
+    private void getModelSnapshot(RevertModelSnapshotAction.Request request, JobResultsProvider provider, Consumer<ModelSnapshot> handler,
                                   Consumer<Exception> errorHandler) {
         logger.info("Reverting to snapshot '" + request.getSnapshotId() + "'");
 
@@ -134,7 +138,7 @@ private ActionListener<RevertModelSnapshotAction.Response> wrapRevertDataCountsL
 
 
         return ActionListener.wrap(response -> {
-            jobProvider.dataCounts(jobId, counts -> {
+            jobResultsProvider.dataCounts(jobId, counts -> {
                 counts.setLatestRecordTimeStamp(modelSnapshot.getLatestRecordTimeStamp());
                 jobDataCountsPersister.persistDataCounts(jobId, counts, new ActionListener<Boolean>() {
                     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java
index c7c9488c26825..adb3d35765c03 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java
@@ -16,20 +16,20 @@
 import org.elasticsearch.xpack.core.ml.action.PutCalendarAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateCalendarJobAction;
 import org.elasticsearch.xpack.ml.job.JobManager;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.util.Set;
 
 public class TransportUpdateCalendarJobAction extends HandledTransportAction<UpdateCalendarJobAction.Request, PutCalendarAction.Response> {
 
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final JobManager jobManager;
 
     @Inject
     public TransportUpdateCalendarJobAction(Settings settings, TransportService transportService,
-                                            ActionFilters actionFilters, JobProvider jobProvider, JobManager jobManager) {
+                                            ActionFilters actionFilters, JobResultsProvider jobResultsProvider, JobManager jobManager) {
         super(settings, UpdateCalendarJobAction.NAME, transportService, actionFilters, UpdateCalendarJobAction.Request::new);
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.jobManager = jobManager;
     }
 
@@ -38,7 +38,7 @@ protected void doExecute(Task task, UpdateCalendarJobAction.Request request, Act
         Set<String> jobIdsToAdd = Strings.tokenizeByCommaToSet(request.getJobIdsToAddExpression());
         Set<String> jobIdsToRemove = Strings.tokenizeByCommaToSet(request.getJobIdsToRemoveExpression());
 
-        jobProvider.updateCalendar(request.getCalendarId(), jobIdsToAdd, jobIdsToRemove,
+        jobResultsProvider.updateCalendar(request.getCalendarId(), jobIdsToAdd, jobIdsToRemove,
                 c -> {
                     jobManager.updateProcessOnCalendarChanged(c.getJobIds());
                     listener.onResponse(new PutCalendarAction.Response(c));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java
index 8000eaacd4fbe..9517bedc9a0f0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.results.Result;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.io.IOException;
 import java.util.function.Consumer;
@@ -38,15 +38,15 @@
 public class TransportUpdateModelSnapshotAction extends HandledTransportAction<UpdateModelSnapshotAction.Request,
         UpdateModelSnapshotAction.Response> {
 
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final Client client;
 
     @Inject
     public TransportUpdateModelSnapshotAction(Settings settings, TransportService transportService,
-                                              ActionFilters actionFilters, JobProvider jobProvider, Client client) {
+                                              ActionFilters actionFilters, JobResultsProvider jobResultsProvider, Client client) {
         super(settings, UpdateModelSnapshotAction.NAME, transportService, actionFilters,
             UpdateModelSnapshotAction.Request::new);
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.client = client;
     }
 
@@ -54,7 +54,7 @@ public TransportUpdateModelSnapshotAction(Settings settings, TransportService tr
     protected void doExecute(Task task, UpdateModelSnapshotAction.Request request,
                              ActionListener<UpdateModelSnapshotAction.Response> listener) {
         logger.debug("Received request to update model snapshot [{}] for job [{}]", request.getSnapshotId(), request.getJobId());
-        jobProvider.getModelSnapshot(request.getJobId(), request.getSnapshotId(), modelSnapshot -> {
+        jobResultsProvider.getModelSnapshot(request.getJobId(), request.getSnapshotId(), modelSnapshot -> {
             if (modelSnapshot == null) {
                 listener.onFailure(new ResourceNotFoundException(Messages.getMessage(
                         Messages.REST_NO_SUCH_MODEL_SNAPSHOT, request.getSnapshotId(), request.getJobId())));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java
index ecc6573e7bb87..efe332346efec 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.Result;
 import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
 
 import java.util.Collections;
@@ -29,13 +29,13 @@
 public class DatafeedJobBuilder {
 
     private final Client client;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final Auditor auditor;
     private final Supplier<Long> currentTimeSupplier;
 
-    public DatafeedJobBuilder(Client client, JobProvider jobProvider, Auditor auditor, Supplier<Long> currentTimeSupplier) {
+    public DatafeedJobBuilder(Client client, JobResultsProvider jobResultsProvider, Auditor auditor, Supplier<Long> currentTimeSupplier) {
         this.client = client;
-        this.jobProvider = Objects.requireNonNull(jobProvider);
+        this.jobResultsProvider = Objects.requireNonNull(jobResultsProvider);
         this.auditor = Objects.requireNonNull(auditor);
         this.currentTimeSupplier = Objects.requireNonNull(currentTimeSupplier);
     }
@@ -79,7 +79,7 @@ void build(Job job, DatafeedConfig datafeed, ActionListener<DatafeedJob> listene
                 TimeValue bucketSpan = job.getAnalysisConfig().getBucketSpan();
                 context.latestFinalBucketEndMs = buckets.results().get(0).getTimestamp().getTime() + bucketSpan.millis() - 1;
             }
-            jobProvider.dataCounts(job.getId(), dataCountsHandler, listener::onFailure);
+            jobResultsProvider.dataCounts(job.getId(), dataCountsHandler, listener::onFailure);
         };
 
         // Step 1. Collect latest bucket
@@ -87,7 +87,7 @@ void build(Job job, DatafeedConfig datafeed, ActionListener<DatafeedJob> listene
                 .sortField(Result.TIMESTAMP.getPreferredName())
                 .sortDescending(true).size(1)
                 .includeInterim(false);
-        jobProvider.bucketsViaInternalClient(job.getId(), latestBucketQuery, bucketsHandler, e -> {
+        jobResultsProvider.bucketsViaInternalClient(job.getId(), latestBucketQuery, bucketsHandler, e -> {
             if (e instanceof ResourceNotFoundException) {
                 QueryPage<Bucket> empty = new QueryPage<>(Collections.emptyList(), 0, Bucket.RESULT_TYPE_FIELD);
                 bucketsHandler.accept(empty);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
index fcc9151b755a6..c18c2c940a4c6 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
@@ -52,7 +52,7 @@
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzer;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
 import org.elasticsearch.xpack.ml.job.process.autodetect.UpdateParams;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
@@ -86,7 +86,7 @@ public class JobManager extends AbstractComponent {
             new DeprecationLogger(Loggers.getLogger(JobManager.class));
 
     private final Environment environment;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final ClusterService clusterService;
     private final Auditor auditor;
     private final Client client;
@@ -97,11 +97,12 @@ public class JobManager extends AbstractComponent {
     /**
      * Create a JobManager
      */
-    public JobManager(Environment environment, Settings settings, JobProvider jobProvider, ClusterService clusterService, Auditor auditor,
+    public JobManager(Environment environment, Settings settings, JobResultsProvider jobResultsProvider,
+                      ClusterService clusterService, Auditor auditor,
                       Client client, UpdateJobProcessNotifier updateJobProcessNotifier) {
         super(settings);
         this.environment = environment;
-        this.jobProvider = Objects.requireNonNull(jobProvider);
+        this.jobResultsProvider = Objects.requireNonNull(jobResultsProvider);
         this.clusterService = Objects.requireNonNull(clusterService);
         this.auditor = Objects.requireNonNull(auditor);
         this.client = Objects.requireNonNull(client);
@@ -167,11 +168,6 @@ public QueryPage<Job> expandJobs(String expression, boolean allowNoJobs, Cluster
         return new QueryPage<>(jobs, jobs.size(), Job.RESULTS_FIELD);
     }
 
-    public JobState getJobState(String jobId) {
-        PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-        return MlTasks.getJobState(jobId, tasks);
-    }
-
     /**
      * Validate the char filter/tokenizer/token filter names used in the categorization analyzer config (if any).
      * This validation has to be done server-side; it cannot be done in a client as that won't have loaded the
@@ -249,12 +245,12 @@ public void onFailure(Exception e) {
 
         ActionListener<Boolean> checkForLeftOverDocs = ActionListener.wrap(
                 response -> {
-                    jobProvider.createJobResultIndex(job, state, putJobListener);
+                    jobResultsProvider.createJobResultIndex(job, state, putJobListener);
                 },
                 actionListener::onFailure
         );
 
-        jobProvider.checkForLeftOverDocuments(job, checkForLeftOverDocs);
+        jobResultsProvider.checkForLeftOverDocuments(job, checkForLeftOverDocs);
     }
 
     public void updateJob(UpdateJobAction.Request request, ActionListener<PutJobAction.Response> actionListener) {
@@ -275,14 +271,14 @@ private void validate(JobUpdate jobUpdate, Job job, ActionListener<Void> handler
     private void validateModelSnapshotIdUpdate(Job job, String modelSnapshotId, ChainTaskExecutor chainTaskExecutor) {
         if (modelSnapshotId != null) {
             chainTaskExecutor.add(listener -> {
-                jobProvider.getModelSnapshot(job.getId(), modelSnapshotId, newModelSnapshot -> {
+                jobResultsProvider.getModelSnapshot(job.getId(), modelSnapshotId, newModelSnapshot -> {
                     if (newModelSnapshot == null) {
                         String message = Messages.getMessage(Messages.REST_NO_SUCH_MODEL_SNAPSHOT, modelSnapshotId,
                                 job.getId());
                         listener.onFailure(new ResourceNotFoundException(message));
                         return;
                     }
-                    jobProvider.getModelSnapshot(job.getId(), job.getModelSnapshotId(), oldModelSnapshot -> {
+                    jobResultsProvider.getModelSnapshot(job.getId(), job.getModelSnapshotId(), oldModelSnapshot -> {
                         if (oldModelSnapshot != null
                                 && newModelSnapshot.result.getTimestamp().before(oldModelSnapshot.result.getTimestamp())) {
                             String message = "Job [" + job.getId() + "] has a more recent model snapshot [" +
@@ -307,7 +303,7 @@ private void validateAnalysisLimitsUpdate(Job job, AnalysisLimits newLimits, Cha
                         + " while the job is open"));
                 return;
             }
-            jobProvider.modelSizeStats(job.getId(), modelSizeStats -> {
+            jobResultsProvider.modelSizeStats(job.getId(), modelSizeStats -> {
                 if (modelSizeStats != null) {
                     ByteSizeValue modelSize = new ByteSizeValue(modelSizeStats.getModelBytes(), ByteSizeUnit.BYTES);
                     if (newModelMemoryLimit < modelSize.getMb()) {
@@ -539,7 +535,7 @@ public ClusterState execute(ClusterState currentState) {
 
         // Step 2. Remove the job from any calendars
         CheckedConsumer<Boolean, Exception> removeFromCalendarsHandler = response -> {
-            jobProvider.removeJobFromCalendars(jobId, ActionListener.<Boolean>wrap(deleteJobStateHandler::accept,
+            jobResultsProvider.removeJobFromCalendars(jobId, ActionListener.<Boolean>wrap(deleteJobStateHandler::accept,
                     actionListener::onFailure ));
         };
 
@@ -607,7 +603,7 @@ public ClusterState execute(ClusterState currentState) {
 
         // Step 0. Find the appropriate established model memory for the reverted job
         // -------
-        jobProvider.getEstablishedMemoryUsage(request.getJobId(), modelSizeStats.getTimestamp(), modelSizeStats, clusterStateHandler,
+        jobResultsProvider.getEstablishedMemoryUsage(request.getJobId(), modelSizeStats.getTimestamp(), modelSizeStats, clusterStateHandler,
                 actionListener::onFailure);
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
similarity index 99%
rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java
rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
index 7513cb5a5bbc0..e850d737d3183 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
@@ -129,8 +129,8 @@
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin;
 
-public class JobProvider {
-    private static final Logger LOGGER = Loggers.getLogger(JobProvider.class);
+public class JobResultsProvider {
+    private static final Logger LOGGER = Loggers.getLogger(JobResultsProvider.class);
 
     private static final int RECORDS_SIZE_PARAM = 10000;
     public static final int BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE = 20;
@@ -139,7 +139,7 @@ public class JobProvider {
     private final Client client;
     private final Settings settings;
 
-    public JobProvider(Client client, Settings settings) {
+    public JobResultsProvider(Client client, Settings settings) {
         this.client = Objects.requireNonNull(client);
         this.settings = settings;
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/overallbuckets/OverallBucketsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/overallbuckets/OverallBucketsProvider.java
index 9bcabad9aea5b..d6ade87fa6e7b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/overallbuckets/OverallBucketsProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/overallbuckets/OverallBucketsProvider.java
@@ -71,7 +71,7 @@ private static Date getHistogramBucketTimestamp(Histogram.Bucket bucket) {
     static class TopNScores extends PriorityQueue<Double> {
 
         TopNScores(int n) {
-            super(n, false);
+            super(n);
         }
 
         @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java
index 43c3f4825ddf3..0b9cb833c8980 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java
@@ -30,6 +30,11 @@
 public class NativeController {
     private static final Logger LOGGER = Loggers.getLogger(NativeController.class);
 
+    /**
+     * Process controller native program name
+     */
+    private static final String CONTROLLER = "controller";
+
     // The controller process should already be running by the time this class tries to connect to it, so the timeout
     // can be short (although there's a gotcha with EBS volumes restored from snapshot, so not TOO short)
     private static final Duration CONTROLLER_CONNECT_TIMEOUT = Duration.ofSeconds(10);
@@ -50,7 +55,7 @@ public class NativeController {
     private final OutputStream commandStream;
 
     NativeController(Environment env, NamedPipeHelper namedPipeHelper) throws IOException {
-        ProcessPipes processPipes = new ProcessPipes(env, namedPipeHelper, ProcessCtrl.CONTROLLER, null,
+        ProcessPipes processPipes = new ProcessPipes(env, namedPipeHelper, CONTROLLER, null,
                 true, true, false, false, false, false);
         processPipes.connectStreams(CONTROLLER_CONNECT_TIMEOUT);
         cppLogHandler = new CppLogMessageHandler(null, processPipes.getLogStream().get());
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessBuilderUtils.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessBuilderUtils.java
new file mode 100644
index 0000000000000..fa5488eae011c
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessBuilderUtils.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job.process;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.XPackPlugin;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+public final class ProcessBuilderUtils {
+
+    private ProcessBuilderUtils() {}
+
+    /**
+     * Name of the model config file
+     */
+    public static final String ML_MODEL_CONF = "mlmodel.conf";
+
+    public static <T> void addIfNotNull(T object, String argKey, List<String> command) {
+        if (object != null) {
+            String param = argKey + object;
+            command.add(param);
+        }
+    }
+
+    public static void addIfNotNull(TimeValue timeValue, String argKey, List<String> command) {
+        addIfNotNull(timeValue == null ? null : timeValue.getSeconds(), argKey, command);
+    }
+
+    /**
+     * Return true if there is a file ES_HOME/config/mlmodel.conf
+     */
+    public static boolean modelConfigFilePresent(Environment env) {
+        Path modelConfPath = XPackPlugin.resolveConfigFile(env, ML_MODEL_CONF);
+
+        return Files.isRegularFile(modelConfPath);
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrl.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrl.java
deleted file mode 100644
index 35033fcfe8430..0000000000000
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrl.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml.job.process;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.xpack.core.XPackPlugin;
-import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
-import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
-import org.elasticsearch.xpack.core.ml.job.config.Job;
-
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.stream.Collectors;
-
-
-/**
- * Utility class for running a Ml process<br>
- * The process runs in a clean environment.
- */
-public class ProcessCtrl {
-
-    /**
-     * Autodetect API native program name - always loaded from the same directory as the controller process
-     */
-    public static final String AUTODETECT = "autodetect";
-    static final String AUTODETECT_PATH = "./" + AUTODETECT;
-
-    /**
-     * The normalization native program name - always loaded from the same directory as the controller process
-     */
-    public static final String NORMALIZE = "normalize";
-    static final String NORMALIZE_PATH = "./" + NORMALIZE;
-
-    /**
-     * Process controller native program name
-     */
-    public static final String CONTROLLER = "controller";
-
-    /**
-     * Name of the config setting containing the path to the logs directory
-     */
-    private static final int DEFAULT_MAX_NUM_RECORDS = 500;
-    /**
-     * The maximum number of anomaly records that will be written each bucket
-     */
-    public static final Setting<Integer> MAX_ANOMALY_RECORDS_SETTING = Setting.intSetting("max.anomaly.records", DEFAULT_MAX_NUM_RECORDS,
-            Property.NodeScope);
-
-    /*
-     * General arguments
-     */
-    static final String JOB_ID_ARG = "--jobid=";
-
-    /*
-     * Arguments used by both autodetect and normalize
-     */
-    static final String BUCKET_SPAN_ARG = "--bucketspan=";
-    public static final String DELETE_STATE_FILES_ARG = "--deleteStateFiles";
-    static final String LENGTH_ENCODED_INPUT_ARG = "--lengthEncodedInput";
-    static final String MODEL_CONFIG_ARG = "--modelconfig=";
-    public static final String QUANTILES_STATE_PATH_ARG = "--quantilesState=";
-    static final String MULTIPLE_BUCKET_SPANS_ARG = "--multipleBucketspans=";
-    static final String PER_PARTITION_NORMALIZATION = "--perPartitionNormalization";
-
-    /*
-     * Arguments used by autodetect
-     */
-    static final String LATENCY_ARG = "--latency=";
-    static final String RESULT_FINALIZATION_WINDOW_ARG = "--resultFinalizationWindow=";
-    static final String MULTIVARIATE_BY_FIELDS_ARG = "--multivariateByFields";
-    static final String PERSIST_INTERVAL_ARG = "--persistInterval=";
-    static final String MAX_QUANTILE_INTERVAL_ARG = "--maxQuantileInterval=";
-    static final String SUMMARY_COUNT_FIELD_ARG = "--summarycountfield=";
-    static final String TIME_FIELD_ARG = "--timefield=";
-
-    private static final int SECONDS_IN_HOUR = 3600;
-
-    /**
-     * Roughly how often should the C++ process persist state?  A staggering
-     * factor that varies by job is added to this.
-     */
-    static final long DEFAULT_BASE_PERSIST_INTERVAL = 10800; // 3 hours
-
-    /**
-     * Roughly how often should the C++ process output quantiles when no
-     * anomalies are being detected?  A staggering factor that varies by job is
-     * added to this.
-     */
-    static final int BASE_MAX_QUANTILE_INTERVAL = 21600; // 6 hours
-
-    /**
-     * Name of the model config file
-     */
-    static final String ML_MODEL_CONF = "mlmodel.conf";
-
-    /**
-     * Persisted quantiles are written to disk so they can be read by
-     * the autodetect program.  All quantiles files have this extension.
-     */
-    private static final String QUANTILES_FILE_EXTENSION = ".json";
-
-    /**
-     * Config setting storing the flag that disables model persistence
-     */
-    public static final Setting<Boolean> DONT_PERSIST_MODEL_STATE_SETTING = Setting.boolSetting("no.model.state.persist", false,
-            Property.NodeScope);
-
-    static String maxAnomalyRecordsArg(Settings settings) {
-        return "--maxAnomalyRecords=" + MAX_ANOMALY_RECORDS_SETTING.get(settings);
-    }
-
-    private ProcessCtrl() {
-    }
-
-    /**
-     * This random time of up to 1 hour is added to intervals at which we
-     * tell the C++ process to perform periodic operations.  This means that
-     * when there are many jobs there is a certain amount of staggering of
-     * their periodic operations.  A given job will always be given the same
-     * staggering interval (for a given JVM implementation).
-     *
-     * @param jobId The ID of the job to calculate the staggering interval for
-     * @return The staggering interval
-     */
-    static int calculateStaggeringInterval(String jobId) {
-        Random rng = new Random(jobId.hashCode());
-        return rng.nextInt(SECONDS_IN_HOUR);
-    }
-
-    public static List<String> buildAutodetectCommand(Environment env, Settings settings, Job job, Logger logger) {
-        List<String> command = new ArrayList<>();
-        command.add(AUTODETECT_PATH);
-
-        String jobId = JOB_ID_ARG + job.getId();
-        command.add(jobId);
-
-        AnalysisConfig analysisConfig = job.getAnalysisConfig();
-        if (analysisConfig != null) {
-            addIfNotNull(analysisConfig.getBucketSpan(), BUCKET_SPAN_ARG, command);
-            addIfNotNull(analysisConfig.getLatency(), LATENCY_ARG, command);
-            addIfNotNull(analysisConfig.getSummaryCountFieldName(),
-                    SUMMARY_COUNT_FIELD_ARG, command);
-            addIfNotNull(analysisConfig.getMultipleBucketSpans(),
-                    MULTIPLE_BUCKET_SPANS_ARG, command);
-            if (Boolean.TRUE.equals(analysisConfig.getOverlappingBuckets())) {
-                Long window = analysisConfig.getResultFinalizationWindow();
-                if (window == null) {
-                    window = AnalysisConfig.DEFAULT_RESULT_FINALIZATION_WINDOW;
-                }
-                command.add(RESULT_FINALIZATION_WINDOW_ARG + window);
-            }
-            if (Boolean.TRUE.equals(analysisConfig.getMultivariateByFields())) {
-                command.add(MULTIVARIATE_BY_FIELDS_ARG);
-            }
-
-            if (analysisConfig.getUsePerPartitionNormalization()) {
-                command.add(PER_PARTITION_NORMALIZATION);
-            }
-        }
-
-        // Input is always length encoded
-        command.add(LENGTH_ENCODED_INPUT_ARG);
-
-        // Limit the number of output records
-        command.add(maxAnomalyRecordsArg(settings));
-
-        // always set the time field
-        String timeFieldArg = TIME_FIELD_ARG + getTimeFieldOrDefault(job);
-        command.add(timeFieldArg);
-
-        int intervalStagger = calculateStaggeringInterval(job.getId());
-        logger.debug("Periodic operations staggered by " + intervalStagger +" seconds for job '" + job.getId() + "'");
-
-        // Supply a URL for persisting/restoring model state unless model
-        // persistence has been explicitly disabled.
-        if (DONT_PERSIST_MODEL_STATE_SETTING.get(settings)) {
-            logger.info("Will not persist model state - "  + DONT_PERSIST_MODEL_STATE_SETTING + " setting was set");
-        } else {
-            // Persist model state every few hours even if the job isn't closed
-            long persistInterval = (job.getBackgroundPersistInterval() == null) ?
-                    (DEFAULT_BASE_PERSIST_INTERVAL + intervalStagger) :
-                        job.getBackgroundPersistInterval().getSeconds();
-                    command.add(PERSIST_INTERVAL_ARG + persistInterval);
-        }
-
-        int maxQuantileInterval = BASE_MAX_QUANTILE_INTERVAL + intervalStagger;
-        command.add(MAX_QUANTILE_INTERVAL_ARG + maxQuantileInterval);
-
-        if (modelConfigFilePresent(env)) {
-            String modelConfigFile = XPackPlugin.resolveConfigFile(env, ML_MODEL_CONF).toString();
-            command.add(MODEL_CONFIG_ARG + modelConfigFile);
-        }
-
-        return command;
-    }
-
-    private static String getTimeFieldOrDefault(Job job) {
-        DataDescription dataDescription = job.getDataDescription();
-        boolean useDefault = dataDescription == null
-                || Strings.isNullOrEmpty(dataDescription.getTimeField());
-        return useDefault ? DataDescription.DEFAULT_TIME_FIELD : dataDescription.getTimeField();
-    }
-
-    private static void addIfNotNull(TimeValue timeValue, String argKey, List<String> command) {
-        addIfNotNull(timeValue == null ? null : timeValue.getSeconds(), argKey, command);
-    }
-
-    private static void addIfNotNull(List<TimeValue> timeValues, String argKey, List<String> command) {
-        if (timeValues != null) {
-            addIfNotNull(timeValues.stream().map(TimeValue::getSeconds).collect(Collectors.toList()), argKey, command);
-        }
-    }
-
-    private static <T> void addIfNotNull(T object, String argKey, List<String> command) {
-        if (object != null) {
-            String param = argKey + object;
-            command.add(param);
-        }
-    }
-
-    /**
-     * Return true if there is a file ES_HOME/config/mlmodel.conf
-     */
-    public static boolean modelConfigFilePresent(Environment env) {
-        Path modelConfPath = XPackPlugin.resolveConfigFile(env, ML_MODEL_CONF);
-
-        return Files.isRegularFile(modelConfPath);
-    }
-
-    /**
-     * Build the command to start the normalizer process.
-     */
-    public static List<String> buildNormalizerCommand(Environment env, String jobId, String quantilesState, Integer bucketSpan,
-            boolean perPartitionNormalization) throws IOException {
-
-        List<String> command = new ArrayList<>();
-        command.add(NORMALIZE_PATH);
-        addIfNotNull(bucketSpan, BUCKET_SPAN_ARG, command);
-        command.add(LENGTH_ENCODED_INPUT_ARG);
-        if (perPartitionNormalization) {
-            command.add(PER_PARTITION_NORMALIZATION);
-        }
-
-        if (quantilesState != null) {
-            Path quantilesStateFilePath = writeNormalizerInitState(jobId, quantilesState, env);
-
-            String stateFileArg = QUANTILES_STATE_PATH_ARG + quantilesStateFilePath;
-            command.add(stateFileArg);
-            command.add(DELETE_STATE_FILES_ARG);
-        }
-
-        if (modelConfigFilePresent(env)) {
-            String modelConfigFile = XPackPlugin.resolveConfigFile(env, ML_MODEL_CONF).toString();
-            command.add(MODEL_CONFIG_ARG + modelConfigFile);
-        }
-
-        return command;
-    }
-
-    /**
-     * Write the normalizer init state to file.
-     */
-    public static Path writeNormalizerInitState(String jobId, String state, Environment env)
-            throws IOException {
-        // createTempFile has a race condition where it may return the same
-        // temporary file name to different threads if called simultaneously
-        // from multiple threads, hence add the thread ID to avoid this
-        Path stateFile = Files.createTempFile(env.tmpFile(), jobId + "_quantiles_" + Thread.currentThread().getId(),
-                QUANTILES_FILE_EXTENSION);
-
-        try (BufferedWriter osw = Files.newBufferedWriter(stateFile, StandardCharsets.UTF_8)) {
-            osw.write(state);
-        }
-
-        return stateFile;
-    }
-}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java
index 89418f9d53125..200cb08512572 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java
@@ -6,51 +6,124 @@
 package org.elasticsearch.xpack.ml.job.process.autodetect;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent;
+import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
+import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.MlFilter;
 import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
 import org.elasticsearch.xpack.ml.job.process.NativeController;
-import org.elasticsearch.xpack.ml.job.process.ProcessCtrl;
+import org.elasticsearch.xpack.ml.job.process.ProcessBuilderUtils;
 import org.elasticsearch.xpack.ml.job.process.ProcessPipes;
-import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
 import org.elasticsearch.xpack.ml.job.process.autodetect.writer.AnalysisLimitsWriter;
 import org.elasticsearch.xpack.ml.job.process.autodetect.writer.FieldConfigWriter;
 import org.elasticsearch.xpack.ml.job.process.autodetect.writer.ModelPlotConfigWriter;
 
+import java.io.BufferedWriter;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Random;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.ml.job.process.ProcessBuilderUtils.addIfNotNull;
+
 /**
  * The autodetect process builder.
  */
 public class AutodetectBuilder {
+
+    /**
+     * Autodetect API native program name - always loaded from the same directory as the controller process
+     */
+    public static final String AUTODETECT = "autodetect";
+    static final String AUTODETECT_PATH = "./" + AUTODETECT;
+
+    /*
+     * Arguments used by both autodetect and normalize
+     */
+    public static final String BUCKET_SPAN_ARG = "--bucketspan=";
+    public static final String DELETE_STATE_FILES_ARG = "--deleteStateFiles";
+    public static final String LENGTH_ENCODED_INPUT_ARG = "--lengthEncodedInput";
+    public static final String MODEL_CONFIG_ARG = "--modelconfig=";
+    public static final String QUANTILES_STATE_PATH_ARG = "--quantilesState=";
+    public static final String PER_PARTITION_NORMALIZATION = "--perPartitionNormalization";
+
     private static final String CONF_EXTENSION = ".conf";
+    static final String JOB_ID_ARG = "--jobid=";
     private static final String LIMIT_CONFIG_ARG = "--limitconfig=";
     private static final String MODEL_PLOT_CONFIG_ARG = "--modelplotconfig=";
     private static final String FIELD_CONFIG_ARG = "--fieldconfig=";
+    static final String LATENCY_ARG = "--latency=";
+    static final String RESULT_FINALIZATION_WINDOW_ARG = "--resultFinalizationWindow=";
+    static final String MULTIVARIATE_BY_FIELDS_ARG = "--multivariateByFields";
+    static final String PERSIST_INTERVAL_ARG = "--persistInterval=";
+    static final String MAX_QUANTILE_INTERVAL_ARG = "--maxQuantileInterval=";
+    static final String SUMMARY_COUNT_FIELD_ARG = "--summarycountfield=";
+    static final String TIME_FIELD_ARG = "--timefield=";
+
+    /**
+     * Name of the config setting containing the path to the logs directory
+     */
+    private static final int DEFAULT_MAX_NUM_RECORDS = 500;
+
+    /**
+     * The maximum number of anomaly records that will be written each bucket
+     */
+    public static final Setting<Integer> MAX_ANOMALY_RECORDS_SETTING = Setting.intSetting("max.anomaly.records", DEFAULT_MAX_NUM_RECORDS,
+            Setting.Property.NodeScope);
+
+    /**
+     * Config setting storing the flag that disables model persistence
+     */
+    public static final Setting<Boolean> DONT_PERSIST_MODEL_STATE_SETTING = Setting.boolSetting("no.model.state.persist", false,
+            Setting.Property.NodeScope);
+
+    private static final int SECONDS_IN_HOUR = 3600;
+
+    /**
+     * Roughly how often should the C++ process persist state?  A staggering
+     * factor that varies by job is added to this.
+     */
+    private static final long DEFAULT_BASE_PERSIST_INTERVAL = 10800; // 3 hours
 
-    private Job job;
-    private List<Path> filesToDelete;
-    private Logger logger;
+    /**
+     * Roughly how often should the C++ process output quantiles when no
+     * anomalies are being detected?  A staggering factor that varies by job is
+     * added to this.
+     */
+    static final int BASE_MAX_QUANTILE_INTERVAL = 21600; // 6 hours
+
+    /**
+     * Persisted quantiles are written to disk so they can be read by
+     * the autodetect program.  All quantiles files have this extension.
+     */
+    private static final String QUANTILES_FILE_EXTENSION = ".json";
+
+    private final Job job;
+    private final List<Path> filesToDelete;
+    private final Logger logger;
+    private final Environment env;
+    private final Settings settings;
+    private final NativeController controller;
+    private final ProcessPipes processPipes;
     private Set<MlFilter> referencedFilters;
     private List<ScheduledEvent> scheduledEvents;
     private Quantiles quantiles;
-    private Environment env;
-    private Settings settings;
-    private NativeController controller;
-    private ProcessPipes processPipes;
 
     /**
      * Constructs an autodetect process builder
@@ -98,7 +171,7 @@ public AutodetectBuilder scheduledEvents(List<ScheduledEvent> scheduledEvents) {
      */
     public void build() throws IOException {
 
-        List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job, logger);
+        List<String> command = buildAutodetectCommand();
 
         buildLimits(command);
         buildModelPlotConfig(command);
@@ -109,6 +182,100 @@ public void build() throws IOException {
         controller.startProcess(command);
     }
 
+    /**
+     * Visible for testing
+     */
+    List<String> buildAutodetectCommand() {
+        List<String> command = new ArrayList<>();
+        command.add(AUTODETECT_PATH);
+
+        command.add(JOB_ID_ARG + job.getId());
+
+        AnalysisConfig analysisConfig = job.getAnalysisConfig();
+        if (analysisConfig != null) {
+            addIfNotNull(analysisConfig.getBucketSpan(), BUCKET_SPAN_ARG, command);
+            addIfNotNull(analysisConfig.getLatency(), LATENCY_ARG, command);
+            addIfNotNull(analysisConfig.getSummaryCountFieldName(),
+                    SUMMARY_COUNT_FIELD_ARG, command);
+            if (Boolean.TRUE.equals(analysisConfig.getOverlappingBuckets())) {
+                Long window = analysisConfig.getResultFinalizationWindow();
+                if (window == null) {
+                    window = AnalysisConfig.DEFAULT_RESULT_FINALIZATION_WINDOW;
+                }
+                command.add(RESULT_FINALIZATION_WINDOW_ARG + window);
+            }
+            if (Boolean.TRUE.equals(analysisConfig.getMultivariateByFields())) {
+                command.add(MULTIVARIATE_BY_FIELDS_ARG);
+            }
+
+            if (analysisConfig.getUsePerPartitionNormalization()) {
+                command.add(PER_PARTITION_NORMALIZATION);
+            }
+        }
+
+        // Input is always length encoded
+        command.add(LENGTH_ENCODED_INPUT_ARG);
+
+        // Limit the number of output records
+        command.add(maxAnomalyRecordsArg(settings));
+
+        // always set the time field
+        String timeFieldArg = TIME_FIELD_ARG + getTimeFieldOrDefault(job);
+        command.add(timeFieldArg);
+
+        int intervalStagger = calculateStaggeringInterval(job.getId());
+        logger.debug("[{}] Periodic operations staggered by {} seconds", job.getId(), intervalStagger);
+
+        // Supply a URL for persisting/restoring model state unless model
+        // persistence has been explicitly disabled.
+        if (DONT_PERSIST_MODEL_STATE_SETTING.get(settings)) {
+            logger.info("[{}] Will not persist model state - {} setting was set", job.getId(), DONT_PERSIST_MODEL_STATE_SETTING);
+        } else {
+            // Persist model state every few hours even if the job isn't closed
+            long persistInterval = (job.getBackgroundPersistInterval() == null) ?
+                    (DEFAULT_BASE_PERSIST_INTERVAL + intervalStagger) :
+                    job.getBackgroundPersistInterval().getSeconds();
+            command.add(PERSIST_INTERVAL_ARG + persistInterval);
+        }
+
+        int maxQuantileInterval = BASE_MAX_QUANTILE_INTERVAL + intervalStagger;
+        command.add(MAX_QUANTILE_INTERVAL_ARG + maxQuantileInterval);
+
+        if (ProcessBuilderUtils.modelConfigFilePresent(env)) {
+            String modelConfigFile = XPackPlugin.resolveConfigFile(env, ProcessBuilderUtils.ML_MODEL_CONF).toString();
+            command.add(MODEL_CONFIG_ARG + modelConfigFile);
+        }
+
+        return command;
+    }
+
+
+    static String maxAnomalyRecordsArg(Settings settings) {
+        return "--maxAnomalyRecords=" + MAX_ANOMALY_RECORDS_SETTING.get(settings);
+    }
+
+    private static String getTimeFieldOrDefault(Job job) {
+        DataDescription dataDescription = job.getDataDescription();
+        boolean useDefault = dataDescription == null
+                || Strings.isNullOrEmpty(dataDescription.getTimeField());
+        return useDefault ? DataDescription.DEFAULT_TIME_FIELD : dataDescription.getTimeField();
+    }
+
+    /**
+     * This random time of up to 1 hour is added to intervals at which we
+     * tell the C++ process to perform periodic operations.  This means that
+     * when there are many jobs there is a certain amount of staggering of
+     * their periodic operations.  A given job will always be given the same
+     * staggering interval (for a given JVM implementation).
+     *
+     * @param jobId The ID of the job to calculate the staggering interval for
+     * @return The staggering interval
+     */
+    static int calculateStaggeringInterval(String jobId) {
+        Random rng = new Random(jobId.hashCode());
+        return rng.nextInt(SECONDS_IN_HOUR);
+    }
+
     private void buildLimits(List<String> command) throws IOException {
         if (job.getAnalysisLimits() != null) {
             Path limitConfigFile = Files.createTempFile(env.tmpFile(), "limitconfig", CONF_EXTENSION);
@@ -152,15 +319,32 @@ private void buildQuantiles(List<String> command) throws IOException {
         if (quantiles != null && !quantiles.getQuantileState().isEmpty()) {
             logger.info("Restoring quantiles for job '" + job.getId() + "'");
 
-            Path normalizersStateFilePath = ProcessCtrl.writeNormalizerInitState(
-                    job.getId(), quantiles.getQuantileState(), env);
+            Path normalizersStateFilePath = writeNormalizerInitState(job.getId(), quantiles.getQuantileState(), env);
 
-            String quantilesStateFileArg = ProcessCtrl.QUANTILES_STATE_PATH_ARG + normalizersStateFilePath;
+            String quantilesStateFileArg = QUANTILES_STATE_PATH_ARG + normalizersStateFilePath;
             command.add(quantilesStateFileArg);
-            command.add(ProcessCtrl.DELETE_STATE_FILES_ARG);
+            command.add(DELETE_STATE_FILES_ARG);
         }
     }
 
+    /**
+     * Write the normalizer init state to file.
+     */
+    public static Path writeNormalizerInitState(String jobId, String state, Environment env)
+            throws IOException {
+        // createTempFile has a race condition where it may return the same
+        // temporary file name to different threads if called simultaneously
+        // from multiple threads, hence add the thread ID to avoid this
+        Path stateFile = Files.createTempFile(env.tmpFile(), jobId + "_quantiles_" + Thread.currentThread().getId(),
+                QUANTILES_FILE_EXTENSION);
+
+        try (BufferedWriter osw = Files.newBufferedWriter(stateFile, StandardCharsets.UTF_8)) {
+            osw.write(state);
+        }
+
+        return stateFile;
+    }
+
     private void buildFieldConfig(List<String> command) throws IOException {
         if (job.getAnalysisConfig() != null) {
             // write to a temporary field config file
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
index 0885a8f9d6479..0206bd88245b3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
@@ -270,7 +270,7 @@ public void persistJob(BiConsumer<Void, Exception> handler) {
     }
 
     @Nullable
-    FlushAcknowledgement waitFlushToCompletion(String flushId) {
+    FlushAcknowledgement waitFlushToCompletion(String flushId) throws InterruptedException {
         LOGGER.debug("[{}] waiting for flush", job.getId());
 
         FlushAcknowledgement flushAcknowledgement;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
index 77e7fe1471611..063ab3b49d146 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
@@ -44,7 +44,7 @@
 import org.elasticsearch.xpack.ml.action.TransportOpenJobAction.JobTask;
 import org.elasticsearch.xpack.ml.job.JobManager;
 import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobRenormalizedResultsPersister;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
 import org.elasticsearch.xpack.ml.job.persistence.ScheduledEventsQueryBuilder;
@@ -112,7 +112,7 @@ public class AutodetectProcessManager extends AbstractComponent {
     private final Environment environment;
     private final ThreadPool threadPool;
     private final JobManager jobManager;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final AutodetectProcessFactory autodetectProcessFactory;
     private final NormalizerFactory normalizerFactory;
 
@@ -132,7 +132,7 @@ public class AutodetectProcessManager extends AbstractComponent {
     private final Auditor auditor;
 
     public AutodetectProcessManager(Environment environment, Settings settings, Client client, ThreadPool threadPool,
-                                    JobManager jobManager, JobProvider jobProvider, JobResultsPersister jobResultsPersister,
+                                    JobManager jobManager, JobResultsProvider jobResultsProvider, JobResultsPersister jobResultsPersister,
                                     JobDataCountsPersister jobDataCountsPersister,
                                     AutodetectProcessFactory autodetectProcessFactory, NormalizerFactory normalizerFactory,
                                     NamedXContentRegistry xContentRegistry, Auditor auditor) {
@@ -145,7 +145,7 @@ public AutodetectProcessManager(Environment environment, Settings settings, Clie
         this.autodetectProcessFactory = autodetectProcessFactory;
         this.normalizerFactory = normalizerFactory;
         this.jobManager = jobManager;
-        this.jobProvider = jobProvider;
+        this.jobResultsProvider = jobResultsProvider;
         this.jobResultsPersister = jobResultsPersister;
         this.jobDataCountsPersister = jobDataCountsPersister;
         this.auditor = auditor;
@@ -362,7 +362,7 @@ public void writeUpdateProcessMessage(JobTask jobTask, UpdateParams updateParams
                         Job job = jobManager.getJobOrThrowIfUnknown(jobTask.getJobId());
                         DataCounts dataCounts = getStatistics(jobTask).get().v1();
                         ScheduledEventsQueryBuilder query = new ScheduledEventsQueryBuilder().start(job.earliestValidTimestamp(dataCounts));
-                        jobProvider.scheduledEventsForJob(jobTask.getJobId(), job.getGroups(), query, eventsListener);
+                        jobResultsProvider.scheduledEventsForJob(jobTask.getJobId(), job.getGroups(), query, eventsListener);
                     } else {
                         eventsListener.onResponse(null);
                     }
@@ -403,7 +403,7 @@ public void openJob(JobTask jobTask, Consumer<Exception> handler) {
 
         logger.info("Opening job [{}]", jobId);
         processByAllocation.putIfAbsent(jobTask.getAllocationId(), new ProcessContext(jobTask));
-        jobProvider.getAutodetectParams(job, params -> {
+        jobResultsProvider.getAutodetectParams(job, params -> {
             // We need to fork, otherwise we restore model state from a network thread (several GET api calls):
             threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(new AbstractRunnable() {
                 @Override
@@ -495,7 +495,7 @@ AutodetectCommunicator create(JobTask jobTask, AutodetectParams autodetectParams
         ExecutorService autoDetectExecutorService = threadPool.executor(MachineLearning.AUTODETECT_THREAD_POOL_NAME);
         DataCountsReporter dataCountsReporter = new DataCountsReporter(settings, job, autodetectParams.dataCounts(),
                 jobDataCountsPersister);
-        ScoresUpdater scoresUpdater = new ScoresUpdater(job, jobProvider,
+        ScoresUpdater scoresUpdater = new ScoresUpdater(job, jobResultsProvider,
                 new JobRenormalizedResultsPersister(job.getId(), settings, client), normalizerFactory);
         ExecutorService renormalizerExecutorService = threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME);
         Renormalizer renormalizer = new ShortCircuitingRenormalizer(jobId, scoresUpdater,
@@ -504,7 +504,7 @@ AutodetectCommunicator create(JobTask jobTask, AutodetectParams autodetectParams
         AutodetectProcess process = autodetectProcessFactory.createAutodetectProcess(job, autodetectParams, autoDetectExecutorService,
                 onProcessCrash(jobTask));
         AutoDetectResultProcessor processor = new AutoDetectResultProcessor(
-                client, auditor, jobId, renormalizer, jobResultsPersister, jobProvider, autodetectParams.modelSizeStats(),
+                client, auditor, jobId, renormalizer, jobResultsPersister, jobResultsProvider, autodetectParams.modelSizeStats(),
                 autodetectParams.modelSnapshot() != null);
         ExecutorService autodetectWorkerExecutor;
         try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java
index adaed1f6f7c36..01ad0bec85aec 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java
@@ -6,21 +6,20 @@
 package org.elasticsearch.xpack.ml.job.process.autodetect;
 
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.job.process.NativeController;
-import org.elasticsearch.xpack.ml.job.process.ProcessCtrl;
 import org.elasticsearch.xpack.ml.job.process.ProcessPipes;
 import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResultsParser;
 import org.elasticsearch.xpack.ml.job.process.autodetect.output.StateProcessor;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.utils.NamedPipeHelper;
 
 import java.io.IOException;
@@ -55,8 +54,9 @@ public AutodetectProcess createAutodetectProcess(Job job,
                                                      ExecutorService executorService,
                                                      Runnable onProcessCrash) {
         List<Path> filesToDelete = new ArrayList<>();
-        ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, ProcessCtrl.AUTODETECT, job.getId(),
-                true, false, true, true, params.modelSnapshot() != null, !ProcessCtrl.DONT_PERSIST_MODEL_STATE_SETTING.get(settings));
+        ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, AutodetectBuilder.AUTODETECT, job.getId(),
+                true, false, true, true, params.modelSnapshot() != null,
+                !AutodetectBuilder.DONT_PERSIST_MODEL_STATE_SETTING.get(settings));
         createNativeProcess(job, params, processPipes, filesToDelete);
         boolean includeTokensField = MachineLearning.CATEGORIZATION_TOKENIZATION_IN_JAVA
                 && job.getAnalysisConfig().getCategorizationFieldName() != null;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java
index da5e70112f045..b86ec4de8257f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java
@@ -34,7 +34,7 @@
 import org.elasticsearch.xpack.core.ml.job.results.Influencer;
 import org.elasticsearch.xpack.core.ml.job.results.ModelPlot;
 import org.elasticsearch.xpack.ml.MachineLearning;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess;
 import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer;
@@ -88,7 +88,7 @@ public class AutoDetectResultProcessor {
     private final String jobId;
     private final Renormalizer renormalizer;
     private final JobResultsPersister persister;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final boolean restoredSnapshot;
 
     final CountDownLatch completionLatch = new CountDownLatch(1);
@@ -107,20 +107,22 @@ public class AutoDetectResultProcessor {
     private volatile boolean haveNewLatestModelSizeStats;
     private Future<?> scheduledEstablishedModelMemoryUpdate; // only accessed in synchronized methods
 
-    public AutoDetectResultProcessor(Client client, Auditor auditor, String jobId, Renormalizer renormalizer, JobResultsPersister persister,
-                                     JobProvider jobProvider, ModelSizeStats latestModelSizeStats, boolean restoredSnapshot) {
-        this(client, auditor, jobId, renormalizer, persister, jobProvider, latestModelSizeStats, restoredSnapshot, new FlushListener());
+    public AutoDetectResultProcessor(Client client, Auditor auditor, String jobId, Renormalizer renormalizer,
+                                     JobResultsPersister persister, JobResultsProvider jobResultsProvider,
+                                     ModelSizeStats latestModelSizeStats, boolean restoredSnapshot) {
+        this(client, auditor, jobId, renormalizer, persister, jobResultsProvider, latestModelSizeStats,
+                restoredSnapshot, new FlushListener());
     }
 
     AutoDetectResultProcessor(Client client, Auditor auditor, String jobId, Renormalizer renormalizer, JobResultsPersister persister,
-                              JobProvider jobProvider, ModelSizeStats latestModelSizeStats, boolean restoredSnapshot,
+                              JobResultsProvider jobResultsProvider, ModelSizeStats latestModelSizeStats, boolean restoredSnapshot,
                               FlushListener flushListener) {
         this.client = Objects.requireNonNull(client);
         this.auditor = Objects.requireNonNull(auditor);
         this.jobId = Objects.requireNonNull(jobId);
         this.renormalizer = Objects.requireNonNull(renormalizer);
         this.persister = Objects.requireNonNull(persister);
-        this.jobProvider = Objects.requireNonNull(jobProvider);
+        this.jobResultsProvider = Objects.requireNonNull(jobResultsProvider);
         this.flushListener = Objects.requireNonNull(flushListener);
         this.latestModelSizeStats = Objects.requireNonNull(latestModelSizeStats);
         this.restoredSnapshot = restoredSnapshot;
@@ -214,7 +216,7 @@ void processResult(Context context, AutodetectResult result) {
 
             // if we haven't previously set established model memory, consider trying again after
             // a reasonable number of buckets have elapsed since the last model size stats update
-            long minEstablishedTimespanMs = JobProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE * bucket.getBucketSpan() * 1000L;
+            long minEstablishedTimespanMs = JobResultsProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE * bucket.getBucketSpan() * 1000L;
             if (haveNewLatestModelSizeStats && latestEstablishedModelMemory == 0 && latestDateForEstablishedModelMemoryCalc.getTime()
                 > latestModelSizeStats.getTimestamp().getTime() + minEstablishedTimespanMs) {
                 scheduleEstablishedModelMemoryUpdate(ESTABLISHED_MODEL_MEMORY_UPDATE_DELAY);
@@ -314,7 +316,7 @@ private void processModelSizeStats(Context context, ModelSizeStats modelSizeStat
         // This is a crude way to NOT refresh the index and NOT attempt to update established model memory during the first 20 buckets
         // because this is when the model size stats are likely to be least stable and lots of updates will be coming through, and
         // we'll NEVER consider memory usage to be established during this period
-        if (restoredSnapshot || bucketCount >= JobProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE) {
+        if (restoredSnapshot || bucketCount >= JobResultsProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE) {
             scheduleEstablishedModelMemoryUpdate(ESTABLISHED_MODEL_MEMORY_UPDATE_DELAY);
         }
     }
@@ -429,7 +431,7 @@ private void updateEstablishedModelMemoryOnJob() {
         // We need to make all results written up to and including these stats available for the established memory calculation
         persister.commitResultWrites(jobId);
 
-        jobProvider.getEstablishedMemoryUsage(jobId, latestBucketTimestamp, modelSizeStatsForCalc, establishedModelMemory -> {
+        jobResultsProvider.getEstablishedMemoryUsage(jobId, latestBucketTimestamp, modelSizeStatsForCalc, establishedModelMemory -> {
             if (latestEstablishedModelMemory != establishedModelMemory) {
                 JobUpdate update = new JobUpdate.Builder(jobId).setEstablishedModelMemory(establishedModelMemory).build();
                 UpdateJobAction.Request updateRequest = UpdateJobAction.Request.internal(jobId, update);
@@ -485,7 +487,7 @@ public void awaitCompletion() throws TimeoutException {
      * @return The {@link FlushAcknowledgement} if the flush has completed or the parsing finished; {@code null} if the timeout expired
      */
     @Nullable
-    public FlushAcknowledgement waitForFlushAcknowledgement(String flushId, Duration timeout) {
+    public FlushAcknowledgement waitForFlushAcknowledgement(String flushId, Duration timeout) throws InterruptedException {
         return failed ? null : flushListener.waitForFlush(flushId, timeout);
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushListener.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushListener.java
index 66b43b5e36fa4..1340556fbcdb1 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushListener.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushListener.java
@@ -22,18 +22,14 @@ class FlushListener {
     final AtomicBoolean cleared = new AtomicBoolean(false);
 
     @Nullable
-    FlushAcknowledgement waitForFlush(String flushId, Duration timeout) {
+    FlushAcknowledgement waitForFlush(String flushId, Duration timeout) throws InterruptedException {
         if (cleared.get()) {
             return null;
         }
 
         FlushAcknowledgementHolder holder = awaitingFlushed.computeIfAbsent(flushId, (key) -> new FlushAcknowledgementHolder(flushId));
-        try {
-            if (holder.latch.await(timeout.toMillis(), TimeUnit.MILLISECONDS)) {
-                return holder.flushAcknowledgement;
-            }
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
+        if (holder.latch.await(timeout.toMillis(), TimeUnit.MILLISECONDS)) {
+            return holder.flushAcknowledgement;
         }
         return null;
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketInfluencerNormalizable.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketInfluencerNormalizable.java
index e55e5ac0346d8..8ee5d1ad6e2d3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketInfluencerNormalizable.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketInfluencerNormalizable.java
@@ -46,6 +46,11 @@ public String getPersonFieldName() {
         return bucketInfluencer.getInfluencerFieldName();
     }
 
+    @Override
+    public String getPersonFieldValue() {
+        return null;
+    }
+
     @Override
     public String getFunctionName() {
         return null;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java
index 1ba2e77040897..7ef23cb513b7f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java
@@ -64,6 +64,11 @@ public String getPersonFieldName() {
         return null;
     }
 
+    @Override
+    public String getPersonFieldValue() {
+        return null;
+    }
+
     @Override
     public String getFunctionName() {
         return null;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizable.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizable.java
index 74cb86a3fdfdd..bc1567ac00a14 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizable.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizable.java
@@ -44,6 +44,11 @@ public String getPersonFieldName() {
         return influencer.getInfluencerFieldName();
     }
 
+    @Override
+    public String getPersonFieldValue() {
+        return influencer.getInfluencerFieldValue();
+    }
+
     @Override
     public String getFunctionName() {
         return null;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/MultiplyingNormalizerProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/MultiplyingNormalizerProcess.java
index fc7bd35188473..8aa266e15d22e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/MultiplyingNormalizerProcess.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/MultiplyingNormalizerProcess.java
@@ -63,10 +63,11 @@ public void writeRecord(String[] record) throws IOException {
             result.setPartitionFieldName(record[1]);
             result.setPartitionFieldValue(record[2]);
             result.setPersonFieldName(record[3]);
-            result.setFunctionName(record[4]);
-            result.setValueFieldName(record[5]);
-            result.setProbability(Double.parseDouble(record[6]));
-            result.setNormalizedScore(factor * Double.parseDouble(record[7]));
+            result.setPersonFieldValue(record[4]);
+            result.setFunctionName(record[5]);
+            result.setValueFieldName(record[6]);
+            result.setProbability(Double.parseDouble(record[7]));
+            result.setNormalizedScore(factor * Double.parseDouble(record[8]));
         } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
             throw new IOException("Unable to write to no-op normalizer", e);
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java
index 0c6bb407e7835..c96a3b48fe1d8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java
@@ -9,10 +9,9 @@
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.job.process.NativeController;
-import org.elasticsearch.xpack.ml.job.process.ProcessCtrl;
 import org.elasticsearch.xpack.ml.job.process.ProcessPipes;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.utils.NamedPipeHelper;
 
 import java.io.IOException;
@@ -40,7 +39,7 @@ public NativeNormalizerProcessFactory(Environment env, Settings settings, Native
     @Override
     public NormalizerProcess createNormalizerProcess(String jobId, String quantilesState, Integer bucketSpan,
                                                      boolean perPartitionNormalization, ExecutorService executorService) {
-        ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, ProcessCtrl.NORMALIZE, jobId,
+        ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, NormalizerBuilder.NORMALIZE, jobId,
                 true, false, true, true, false, false);
         createNativeProcess(jobId, quantilesState, processPipes, bucketSpan, perPartitionNormalization);
 
@@ -52,7 +51,7 @@ private void createNativeProcess(String jobId, String quantilesState, ProcessPip
                                      boolean perPartitionNormalization) {
 
         try {
-            List<String> command = ProcessCtrl.buildNormalizerCommand(env, jobId, quantilesState, bucketSpan, perPartitionNormalization);
+            List<String> command = new NormalizerBuilder(env, jobId, quantilesState, bucketSpan, perPartitionNormalization).build();
             processPipes.addArgs(command);
             nativeController.startProcess(command);
             processPipes.connectStreams(PROCESS_STARTUP_TIMEOUT);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizable.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizable.java
index 606be98ae10e7..7efadf2961308 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizable.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizable.java
@@ -44,6 +44,8 @@ public Normalizable(String indexName) {
 
     abstract String getPersonFieldName();
 
+    abstract String getPersonFieldValue();
+
     abstract String getFunctionName();
 
     abstract String getValueFieldName();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java
index 2c929ff4f1ae4..2d4e2135478f3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java
@@ -70,6 +70,7 @@ public void normalize(Integer bucketSpan, boolean perPartitionNormalization,
                     NormalizerResult.PARTITION_FIELD_NAME_FIELD.getPreferredName(),
                     NormalizerResult.PARTITION_FIELD_VALUE_FIELD.getPreferredName(),
                     NormalizerResult.PERSON_FIELD_NAME_FIELD.getPreferredName(),
+                    NormalizerResult.PERSON_FIELD_VALUE_FIELD.getPreferredName(),
                     NormalizerResult.FUNCTION_NAME_FIELD.getPreferredName(),
                     NormalizerResult.VALUE_FIELD_NAME_FIELD.getPreferredName(),
                     NormalizerResult.PROBABILITY_FIELD.getPreferredName(),
@@ -108,6 +109,7 @@ private static void writeNormalizableAndChildrenRecursively(Normalizable normali
                     Strings.coalesceToEmpty(normalizable.getPartitionFieldName()),
                     Strings.coalesceToEmpty(normalizable.getPartitionFieldValue()),
                     Strings.coalesceToEmpty(normalizable.getPersonFieldName()),
+                    Strings.coalesceToEmpty(normalizable.getPersonFieldValue()),
                     Strings.coalesceToEmpty(normalizable.getFunctionName()),
                     Strings.coalesceToEmpty(normalizable.getValueFieldName()),
                     Double.toString(normalizable.getProbability()),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilder.java
new file mode 100644
index 0000000000000..5630a75127506
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilder.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job.process.normalizer;
+
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.ml.job.process.ProcessBuilderUtils;
+import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectBuilder;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.xpack.ml.job.process.ProcessBuilderUtils.addIfNotNull;
+
+public class NormalizerBuilder {
+
+    /**
+     * The normalization native program name - always loaded from the same directory as the controller process
+     */
+    public static final String NORMALIZE = "normalize";
+    static final String NORMALIZE_PATH = "./" + NORMALIZE;
+
+    private final Environment env;
+    private final String jobId;
+    private final String quantilesState;
+    private final Integer bucketSpan;
+    private final boolean perPartitionNormalization;
+
+    public NormalizerBuilder(Environment env, String jobId, String quantilesState, Integer bucketSpan,
+                             boolean perPartitionNormalization) {
+        this.env = env;
+        this.jobId = jobId;
+        this.quantilesState = quantilesState;
+        this.bucketSpan = bucketSpan;
+        this.perPartitionNormalization = perPartitionNormalization;
+    }
+
+    /**
+     * Build the command to start the normalizer process.
+     */
+    public List<String> build() throws IOException {
+
+        List<String> command = new ArrayList<>();
+        command.add(NORMALIZE_PATH);
+        addIfNotNull(bucketSpan, AutodetectBuilder.BUCKET_SPAN_ARG, command);
+        command.add(AutodetectBuilder.LENGTH_ENCODED_INPUT_ARG);
+        if (perPartitionNormalization) {
+            command.add(AutodetectBuilder.PER_PARTITION_NORMALIZATION);
+        }
+
+        if (quantilesState != null) {
+            Path quantilesStateFilePath = AutodetectBuilder.writeNormalizerInitState(jobId, quantilesState, env);
+
+            String stateFileArg = AutodetectBuilder.QUANTILES_STATE_PATH_ARG + quantilesStateFilePath;
+            command.add(stateFileArg);
+            command.add(AutodetectBuilder.DELETE_STATE_FILES_ARG);
+        }
+
+        if (ProcessBuilderUtils.modelConfigFilePresent(env)) {
+            String modelConfigFile = XPackPlugin.resolveConfigFile(env, ProcessBuilderUtils.ML_MODEL_CONF).toString();
+            command.add(AutodetectBuilder.MODEL_CONFIG_ARG + modelConfigFile);
+        }
+
+        return command;
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResult.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResult.java
index 5cb2932a28ba5..269792dbe7797 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResult.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResult.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.job.process.normalizer;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -26,6 +27,7 @@ public class NormalizerResult implements ToXContentObject, Writeable {
     static final ParseField PARTITION_FIELD_NAME_FIELD = new ParseField("partition_field_name");
     static final ParseField PARTITION_FIELD_VALUE_FIELD = new ParseField("partition_field_value");
     static final ParseField PERSON_FIELD_NAME_FIELD = new ParseField("person_field_name");
+    static final ParseField PERSON_FIELD_VALUE_FIELD = new ParseField("person_field_value");
     static final ParseField FUNCTION_NAME_FIELD = new ParseField("function_name");
     static final ParseField VALUE_FIELD_NAME_FIELD = new ParseField("value_field_name");
     static final ParseField PROBABILITY_FIELD = new ParseField("probability");
@@ -39,6 +41,7 @@ public class NormalizerResult implements ToXContentObject, Writeable {
         PARSER.declareString(NormalizerResult::setPartitionFieldName, PARTITION_FIELD_NAME_FIELD);
         PARSER.declareString(NormalizerResult::setPartitionFieldValue, PARTITION_FIELD_VALUE_FIELD);
         PARSER.declareString(NormalizerResult::setPersonFieldName, PERSON_FIELD_NAME_FIELD);
+        PARSER.declareString(NormalizerResult::setPersonFieldValue, PERSON_FIELD_VALUE_FIELD);
         PARSER.declareString(NormalizerResult::setFunctionName, FUNCTION_NAME_FIELD);
         PARSER.declareString(NormalizerResult::setValueFieldName, VALUE_FIELD_NAME_FIELD);
         PARSER.declareDouble(NormalizerResult::setProbability, PROBABILITY_FIELD);
@@ -49,6 +52,7 @@ public class NormalizerResult implements ToXContentObject, Writeable {
     private String partitionFieldName;
     private String partitionFieldValue;
     private String personFieldName;
+    private String personFieldValue;
     private String functionName;
     private String valueFieldName;
     private double probability;
@@ -62,6 +66,9 @@ public NormalizerResult(StreamInput in) throws IOException {
         partitionFieldName = in.readOptionalString();
         partitionFieldValue = in.readOptionalString();
         personFieldName = in.readOptionalString();
+        if (in.getVersion().onOrAfter(Version.V_6_5_0)) {
+            personFieldValue = in.readOptionalString();
+        }
         functionName = in.readOptionalString();
         valueFieldName = in.readOptionalString();
         probability = in.readDouble();
@@ -74,6 +81,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(partitionFieldName);
         out.writeOptionalString(partitionFieldValue);
         out.writeOptionalString(personFieldName);
+        if (out.getVersion().onOrAfter(Version.V_6_5_0)) {
+            out.writeOptionalString(personFieldValue);
+        }
         out.writeOptionalString(functionName);
         out.writeOptionalString(valueFieldName);
         out.writeDouble(probability);
@@ -87,6 +97,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(PARTITION_FIELD_NAME_FIELD.getPreferredName(), partitionFieldName);
         builder.field(PARTITION_FIELD_VALUE_FIELD.getPreferredName(), partitionFieldValue);
         builder.field(PERSON_FIELD_NAME_FIELD.getPreferredName(), personFieldName);
+        builder.field(PERSON_FIELD_VALUE_FIELD.getPreferredName(), personFieldValue);
         builder.field(FUNCTION_NAME_FIELD.getPreferredName(), functionName);
         builder.field(VALUE_FIELD_NAME_FIELD.getPreferredName(), valueFieldName);
         builder.field(PROBABILITY_FIELD.getPreferredName(), probability);
@@ -127,6 +138,14 @@ public void setPersonFieldName(String personFieldName) {
         this.personFieldName = personFieldName;
     }
 
+    public String getPersonFieldValue() {
+        return personFieldValue;
+    }
+
+    public void setPersonFieldValue(String personFieldValue) {
+        this.personFieldValue = personFieldValue;
+    }
+
     public String getFunctionName() {
         return functionName;
     }
@@ -161,7 +180,7 @@ public void setNormalizedScore(double normalizedScore) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(level, partitionFieldName, partitionFieldValue, personFieldName,
+        return Objects.hash(level, partitionFieldName, partitionFieldValue, personFieldName, personFieldValue,
                 functionName, valueFieldName, probability, normalizedScore);
     }
 
@@ -184,6 +203,7 @@ public boolean equals(Object other) {
                 && Objects.equals(this.partitionFieldName, that.partitionFieldName)
                 && Objects.equals(this.partitionFieldValue, that.partitionFieldValue)
                 && Objects.equals(this.personFieldName, that.personFieldName)
+                && Objects.equals(this.personFieldValue, that.personFieldValue)
                 && Objects.equals(this.functionName, that.functionName)
                 && Objects.equals(this.valueFieldName, that.valueFieldName)
                 && this.probability == that.probability
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java
index 4d5d91aa12f8c..91b2a7a505e35 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java
@@ -45,6 +45,11 @@ public String getPersonFieldName() {
         return null;
     }
 
+    @Override
+    public String getPersonFieldValue() {
+        return null;
+    }
+
     @Override
     public String getFunctionName() {
         return null;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/RecordNormalizable.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/RecordNormalizable.java
index 97114130c84d9..f3f32cb526ead 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/RecordNormalizable.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/RecordNormalizable.java
@@ -46,6 +46,12 @@ public String getPersonFieldName() {
         return over != null ? over : record.getByFieldName();
     }
 
+    @Override
+    public String getPersonFieldValue() {
+        String over = record.getOverFieldValue();
+        return over != null ? over : record.getByFieldValue();
+    }
+
     @Override
     public String getFunctionName() {
         return record.getFunction();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java
index ccda255206478..cfb5660c911b5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.xpack.core.ml.job.results.Influencer;
 import org.elasticsearch.xpack.core.ml.job.results.Result;
 import org.elasticsearch.xpack.ml.job.persistence.BatchedDocumentsIterator;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobRenormalizedResultsPersister;
 
 import java.util.ArrayList;
@@ -43,17 +43,17 @@ public class ScoresUpdater {
     private static final long MILLISECONDS_IN_SECOND = 1000;
 
     private final String jobId;
-    private final JobProvider jobProvider;
+    private final JobResultsProvider jobResultsProvider;
     private final JobRenormalizedResultsPersister updatesPersister;
     private final NormalizerFactory normalizerFactory;
     private int bucketSpan;
     private long normalizationWindow;
     private volatile boolean shutdown;
 
-    public ScoresUpdater(Job job, JobProvider jobProvider, JobRenormalizedResultsPersister jobRenormalizedResultsPersister,
+    public ScoresUpdater(Job job, JobResultsProvider jobResultsProvider, JobRenormalizedResultsPersister jobRenormalizedResultsPersister,
                          NormalizerFactory normalizerFactory) {
         jobId = job.getId();
-        this.jobProvider = Objects.requireNonNull(jobProvider);
+        this.jobResultsProvider = Objects.requireNonNull(jobResultsProvider);
         updatesPersister = Objects.requireNonNull(jobRenormalizedResultsPersister);
         this.normalizerFactory = Objects.requireNonNull(normalizerFactory);
         bucketSpan = ((Long) job.getAnalysisConfig().getBucketSpan().seconds()).intValue();
@@ -96,7 +96,7 @@ public void update(String quantilesState, long endBucketEpochMs, long windowExte
     private void updateBuckets(Normalizer normalizer, String quantilesState, long endBucketEpochMs,
                                long windowExtensionMs, int[] counts, boolean perPartitionNormalization) {
         BatchedDocumentsIterator<Result<Bucket>> bucketsIterator =
-                jobProvider.newBatchedBucketsIterator(jobId)
+                jobResultsProvider.newBatchedBucketsIterator(jobId)
                         .timeRange(calcNormalizationWindowStart(endBucketEpochMs, windowExtensionMs), endBucketEpochMs)
                         .includeInterim(false);
 
@@ -145,7 +145,7 @@ private void normalizeBuckets(Normalizer normalizer, List<BucketNormalizable> no
 
     private void updateRecords(Normalizer normalizer, String quantilesState, long endBucketEpochMs,
                                long windowExtensionMs, int[] counts, boolean perPartitionNormalization) {
-        BatchedDocumentsIterator<Result<AnomalyRecord>> recordsIterator = jobProvider.newBatchedRecordsIterator(jobId)
+        BatchedDocumentsIterator<Result<AnomalyRecord>> recordsIterator = jobResultsProvider.newBatchedRecordsIterator(jobId)
                 .timeRange(calcNormalizationWindowStart(endBucketEpochMs, windowExtensionMs), endBucketEpochMs)
                 .includeInterim(false);
 
@@ -168,7 +168,7 @@ private void updateRecords(Normalizer normalizer, String quantilesState, long en
 
     private void updateInfluencers(Normalizer normalizer, String quantilesState, long endBucketEpochMs,
                                    long windowExtensionMs, int[] counts, boolean perPartitionNormalization) {
-        BatchedDocumentsIterator<Result<Influencer>> influencersIterator = jobProvider.newBatchedInfluencersIterator(jobId)
+        BatchedDocumentsIterator<Result<Influencer>> influencersIterator = jobResultsProvider.newBatchedInfluencersIterator(jobId)
                 .timeRange(calcNormalizationWindowStart(endBucketEpochMs, windowExtensionMs), endBucketEpochMs)
                 .includeInterim(false);
 
diff --git a/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy
index 9ba8ea6798b7c..79fced09adb1c 100644
--- a/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy
@@ -9,6 +9,8 @@ grant {
 grant codeBase "${codebase.netty-common}" {
    // for reading the system-wide configuration for the backlog of established sockets
    permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };
 
 grant codeBase "${codebase.netty-transport}" {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java
index 6dd52626f7de1..f37deef12d083 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
 import org.junit.Before;
 
@@ -41,7 +41,7 @@ public class DatafeedJobBuilderTests extends ESTestCase {
 
     private Client client;
     private Auditor auditor;
-    private JobProvider jobProvider;
+    private JobResultsProvider jobResultsProvider;
     private Consumer<Exception> taskHandler;
 
     private DatafeedJobBuilder datafeedJobBuilder;
@@ -54,9 +54,9 @@ public void init() {
         when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
         when(client.settings()).thenReturn(Settings.EMPTY);
         auditor = mock(Auditor.class);
-        jobProvider = mock(JobProvider.class);
+        jobResultsProvider = mock(JobResultsProvider.class);
         taskHandler = mock(Consumer.class);
-        datafeedJobBuilder = new DatafeedJobBuilder(client, jobProvider, auditor, System::currentTimeMillis);
+        datafeedJobBuilder = new DatafeedJobBuilder(client, jobResultsProvider, auditor, System::currentTimeMillis);
 
         Mockito.doAnswer(invocationOnMock -> {
             String jobId = (String) invocationOnMock.getArguments()[0];
@@ -64,14 +64,14 @@ public void init() {
             Consumer<DataCounts> handler = (Consumer<DataCounts>) invocationOnMock.getArguments()[1];
             handler.accept(new DataCounts(jobId));
             return null;
-        }).when(jobProvider).dataCounts(any(), any(), any());
+        }).when(jobResultsProvider).dataCounts(any(), any(), any());
 
         doAnswer(invocationOnMock -> {
             @SuppressWarnings("unchecked")
             Consumer<ResourceNotFoundException> consumer = (Consumer<ResourceNotFoundException>) invocationOnMock.getArguments()[3];
             consumer.accept(new ResourceNotFoundException("dummy"));
             return null;
-        }).when(jobProvider).bucketsViaInternalClient(any(), any(), any(), any());
+        }).when(jobResultsProvider).bucketsViaInternalClient(any(), any(), any(), any());
     }
 
     public void testBuild_GivenScrollDatafeedAndNewJob() throws Exception {
@@ -157,7 +157,7 @@ public void testBuild_GivenBucketsRequestFails() {
             Consumer<Exception> consumer = (Consumer<Exception>) invocationOnMock.getArguments()[3];
             consumer.accept(error);
             return null;
-        }).when(jobProvider).bucketsViaInternalClient(any(), any(), any(), any());
+        }).when(jobResultsProvider).bucketsViaInternalClient(any(), any(), any(), any());
 
         datafeedJobBuilder.build(jobBuilder.build(new Date()), datafeed, ActionListener.wrap(datafeedJob -> fail(), taskHandler));
 
@@ -173,7 +173,7 @@ private void givenLatestTimes(long latestRecordTimestamp, long latestBucketTimes
             dataCounts.setLatestRecordTimeStamp(new Date(latestRecordTimestamp));
             handler.accept(dataCounts);
             return null;
-        }).when(jobProvider).dataCounts(any(), any(), any());
+        }).when(jobResultsProvider).dataCounts(any(), any(), any());
 
         doAnswer(invocationOnMock -> {
             @SuppressWarnings("unchecked")
@@ -183,6 +183,6 @@ private void givenLatestTimes(long latestRecordTimestamp, long latestBucketTimes
             QueryPage<Bucket> bucketQueryPage = new QueryPage<Bucket>(Collections.singletonList(bucket), 1, Bucket.RESULTS_FIELD);
             consumer.accept(bucketQueryPage);
             return null;
-        }).when(jobProvider).bucketsViaInternalClient(any(), any(), any(), any());
+        }).when(jobResultsProvider).bucketsViaInternalClient(any(), any(), any(), any());
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
index 09bb3f7591677..8f6005a216115 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
@@ -36,7 +36,7 @@
 import org.elasticsearch.xpack.ml.MlSingleNodeTestCase;
 import org.elasticsearch.xpack.ml.job.persistence.BucketsQueryBuilder;
 import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
 import org.elasticsearch.xpack.ml.job.persistence.RecordsQueryBuilder;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess;
@@ -72,7 +72,7 @@
 public class AutodetectResultProcessorIT extends MlSingleNodeTestCase {
     private static final String JOB_ID = "autodetect-result-processor-it-job";
 
-    private JobProvider jobProvider;
+    private JobResultsProvider jobResultsProvider;
     private List<ModelSnapshot> capturedUpdateModelSnapshotOnJobRequests;
     private AutoDetectResultProcessor resultProcessor;
     private Renormalizer renormalizer;
@@ -98,11 +98,11 @@ public void createComponents() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(1));
         Auditor auditor = new Auditor(client(), "test_node");
-        jobProvider = new JobProvider(client(), builder.build());
+        jobResultsProvider = new JobResultsProvider(client(), builder.build());
         renormalizer = mock(Renormalizer.class);
         capturedUpdateModelSnapshotOnJobRequests = new ArrayList<>();
         resultProcessor = new AutoDetectResultProcessor(client(), auditor, JOB_ID, renormalizer,
-                new JobResultsPersister(nodeSettings(), client()), jobProvider, new ModelSizeStats.Builder(JOB_ID).build(), false) {
+                new JobResultsPersister(nodeSettings(), client()), jobResultsProvider, new ModelSizeStats.Builder(JOB_ID).build(), false) {
             @Override
             protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) {
                 capturedUpdateModelSnapshotOnJobRequests.add(modelSnapshot);
@@ -159,7 +159,7 @@ public void testProcessResults() throws Exception {
         assertEquals(1, persistedDefinition.count());
         assertEquals(categoryDefinition, persistedDefinition.results().get(0));
 
-        QueryPage<ModelPlot> persistedModelPlot = jobProvider.modelPlot(JOB_ID, 0, 100);
+        QueryPage<ModelPlot> persistedModelPlot = jobResultsProvider.modelPlot(JOB_ID, 0, 100);
         assertEquals(1, persistedModelPlot.count());
         assertEquals(modelPlot, persistedModelPlot.results().get(0));
 
@@ -443,7 +443,7 @@ private QueryPage<Bucket> getBucketQueryPage(BucketsQueryBuilder bucketsQuery) t
         AtomicReference<Exception> errorHolder = new AtomicReference<>();
         AtomicReference<QueryPage<Bucket>> resultHolder = new AtomicReference<>();
         CountDownLatch latch = new CountDownLatch(1);
-        jobProvider.buckets(JOB_ID, bucketsQuery, r -> {
+        jobResultsProvider.buckets(JOB_ID, bucketsQuery, r -> {
             resultHolder.set(r);
             latch.countDown();
         }, e -> {
@@ -461,7 +461,7 @@ private QueryPage<CategoryDefinition> getCategoryDefinition(long categoryId) thr
         AtomicReference<Exception> errorHolder = new AtomicReference<>();
         AtomicReference<QueryPage<CategoryDefinition>> resultHolder = new AtomicReference<>();
         CountDownLatch latch = new CountDownLatch(1);
-        jobProvider.categoryDefinitions(JOB_ID, categoryId, false, null, null, r -> {
+        jobResultsProvider.categoryDefinitions(JOB_ID, categoryId, false, null, null, r -> {
             resultHolder.set(r);
             latch.countDown();
         }, e -> {
@@ -479,7 +479,7 @@ private ModelSizeStats getModelSizeStats() throws Exception {
         AtomicReference<Exception> errorHolder = new AtomicReference<>();
         AtomicReference<ModelSizeStats> resultHolder = new AtomicReference<>();
         CountDownLatch latch = new CountDownLatch(1);
-        jobProvider.modelSizeStats(JOB_ID, modelSizeStats -> {
+        jobResultsProvider.modelSizeStats(JOB_ID, modelSizeStats -> {
             resultHolder.set(modelSizeStats);
             latch.countDown();
         }, e -> {
@@ -497,7 +497,7 @@ private QueryPage<Influencer> getInfluencers() throws Exception {
         AtomicReference<Exception> errorHolder = new AtomicReference<>();
         AtomicReference<QueryPage<Influencer>> resultHolder = new AtomicReference<>();
         CountDownLatch latch = new CountDownLatch(1);
-        jobProvider.influencers(JOB_ID, new InfluencersQueryBuilder().build(), page -> {
+        jobResultsProvider.influencers(JOB_ID, new InfluencersQueryBuilder().build(), page -> {
             resultHolder.set(page);
             latch.countDown();
         }, e -> {
@@ -515,7 +515,7 @@ private QueryPage<AnomalyRecord> getRecords(RecordsQueryBuilder recordsQuery) th
         AtomicReference<Exception> errorHolder = new AtomicReference<>();
         AtomicReference<QueryPage<AnomalyRecord>> resultHolder = new AtomicReference<>();
         CountDownLatch latch = new CountDownLatch(1);
-        jobProvider.records(JOB_ID, recordsQuery, page -> {
+        jobResultsProvider.records(JOB_ID, recordsQuery, page -> {
             resultHolder.set(page);
             latch.countDown();
         }, e -> {
@@ -533,7 +533,7 @@ private QueryPage<ModelSnapshot> getModelSnapshots() throws Exception {
         AtomicReference<Exception> errorHolder = new AtomicReference<>();
         AtomicReference<QueryPage<ModelSnapshot>> resultHolder = new AtomicReference<>();
         CountDownLatch latch = new CountDownLatch(1);
-        jobProvider.modelSnapshots(JOB_ID, 0, 100, page -> {
+        jobResultsProvider.modelSnapshots(JOB_ID, 0, 100, page -> {
             resultHolder.set(page);
             latch.countDown();
         }, e -> {
@@ -551,7 +551,7 @@ private Optional<Quantiles> getQuantiles() throws Exception {
         AtomicReference<Exception> errorHolder = new AtomicReference<>();
         AtomicReference<Optional<Quantiles>> resultHolder = new AtomicReference<>();
         CountDownLatch latch = new CountDownLatch(1);
-        jobProvider.getAutodetectParams(JobTests.buildJobBuilder(JOB_ID).build(),params -> {
+        jobResultsProvider.getAutodetectParams(JobTests.buildJobBuilder(JOB_ID).build(), params -> {
             resultHolder.set(Optional.ofNullable(params.quantiles()));
             latch.countDown();
         }, e -> {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java
index b8e04a8922c5d..df3af13f71403 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
 import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
 import org.junit.Before;
@@ -26,13 +26,13 @@ public class EstablishedMemUsageIT extends BaseMlIntegTestCase {
 
     private long bucketSpan = AnalysisConfig.Builder.DEFAULT_BUCKET_SPAN.getMillis();
 
-    private JobProvider jobProvider;
+    private JobResultsProvider jobResultsProvider;
     private JobResultsPersister jobResultsPersister;
 
     @Before
     public void createComponents() {
         Settings settings = nodeSettings(0);
-        jobProvider = new JobProvider(client(), settings);
+        jobResultsProvider = new JobResultsProvider(client(), settings);
         jobResultsPersister = new JobResultsPersister(settings, client());
     }
 
@@ -251,7 +251,7 @@ private Long queryEstablishedMemoryUsage(String jobId, Integer bucketNum, ModelS
         CountDownLatch latch = new CountDownLatch(1);
 
         Date latestBucketTimestamp = (bucketNum != null) ? new Date(bucketSpan * bucketNum) : null;
-        jobProvider.getEstablishedMemoryUsage(jobId, latestBucketTimestamp, latestModelSizeStats, memUse -> {
+        jobResultsProvider.getEstablishedMemoryUsage(jobId, latestBucketTimestamp, latestModelSizeStats, memUse -> {
                     establishedModelMemoryUsage.set(memUse);
                     latch.countDown();
                 }, e -> {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
similarity index 99%
rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java
rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
index 856b930ac49b5..303fce3d81321 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
@@ -45,7 +45,7 @@
 import org.elasticsearch.xpack.ml.MlSingleNodeTestCase;
 import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder;
 import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
 import org.elasticsearch.xpack.ml.job.persistence.ScheduledEventsQueryBuilder;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams;
@@ -72,9 +72,9 @@
 import static org.hamcrest.core.Is.is;
 
 
-public class JobProviderIT extends MlSingleNodeTestCase {
+public class JobResultsProviderIT extends MlSingleNodeTestCase {
 
-    private JobProvider jobProvider;
+    private JobResultsProvider jobProvider;
 
     @Override
     protected Settings nodeSettings()  {
@@ -95,7 +95,7 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     public void createComponents() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(1));
-        jobProvider = new JobProvider(client(), builder.build());
+        jobProvider = new JobResultsProvider(client(), builder.build());
         waitForMlTemplates();
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java
index e15b5828df96a..a9162cb2ae4df 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java
@@ -34,7 +34,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.MlFilter;
 import org.elasticsearch.xpack.core.ml.job.config.RuleScope;
 import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzerTests;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.process.autodetect.UpdateParams;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
 import org.junit.Before;
@@ -68,7 +68,7 @@ public class JobManagerTests extends ESTestCase {
     private AnalysisRegistry analysisRegistry;
     private Client client;
     private ClusterService clusterService;
-    private JobProvider jobProvider;
+    private JobResultsProvider jobResultsProvider;
     private Auditor auditor;
     private UpdateJobProcessNotifier updateJobProcessNotifier;
 
@@ -79,7 +79,7 @@ public void setup() throws Exception {
         analysisRegistry = CategorizationAnalyzerTests.buildTestAnalysisRegistry(environment);
         client = mock(Client.class);
         clusterService = mock(ClusterService.class);
-        jobProvider = mock(JobProvider.class);
+        jobResultsProvider = mock(JobResultsProvider.class);
         auditor = mock(Auditor.class);
         updateJobProcessNotifier = mock(UpdateJobProcessNotifier.class);
     }
@@ -131,7 +131,7 @@ public void testPutJob_AddsCreateTime() throws IOException {
             ActionListener<Boolean> listener = (ActionListener<Boolean>) invocation.getArguments()[2];
             listener.onResponse(true);
             return null;
-        }).when(jobProvider).createJobResultIndex(requestCaptor.capture(), any(ClusterState.class), any(ActionListener.class));
+        }).when(jobResultsProvider).createJobResultIndex(requestCaptor.capture(), any(ClusterState.class), any(ActionListener.class));
 
         ClusterState clusterState = createClusterState();
 
@@ -404,7 +404,8 @@ private JobManager createJobManager() {
         ClusterSettings clusterSettings = new ClusterSettings(environment.settings(),
                 Collections.singleton(MachineLearningField.MAX_MODEL_MEMORY_LIMIT));
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
-        return new JobManager(environment, environment.settings(), jobProvider, clusterService, auditor, client, updateJobProcessNotifier);
+        return new JobManager(environment, environment.settings(), jobResultsProvider, clusterService,
+                auditor, client, updateJobProcessNotifier);
     }
 
     private ClusterState createClusterState() {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java
similarity index 95%
rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java
rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java
index e33dbc69db607..2c0681c33f22a 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java
@@ -72,7 +72,7 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-public class JobProviderTests extends ESTestCase {
+public class JobResultsProviderTests extends ESTestCase {
     private static final String CLUSTER_NAME = "myCluster";
 
     @SuppressWarnings("unchecked")
@@ -87,7 +87,7 @@ public void testCreateJobResultsIndex() {
         clientBuilder.prepareAlias(resultsIndexName, AnomalyDetectorsIndex.resultsWriteAlias("foo"));
 
         Job.Builder job = buildJobBuilder("foo");
-        JobProvider provider = createProvider(clientBuilder.build());
+        JobResultsProvider provider = createProvider(clientBuilder.build());
         AtomicReference<Boolean> resultHolder = new AtomicReference<>();
 
         ClusterState cs = ClusterState.builder(new ClusterName("_name"))
@@ -141,7 +141,7 @@ public void testCreateJobWithExistingIndex() {
 
         Job.Builder job = buildJobBuilder("foo123");
         job.setResultsIndexName("foo");
-        JobProvider provider = createProvider(clientBuilder.build());
+        JobResultsProvider provider = createProvider(clientBuilder.build());
 
         Index index = mock(Index.class);
         when(index.getName()).thenReturn(AnomalyDetectorsIndex.jobResultsAliasedName("foo"));
@@ -202,7 +202,7 @@ public void testCreateJobRelatedIndicies_createsAliasBecauseIndexNameIsSet() {
         Job.Builder job = buildJobBuilder("foo");
         job.setResultsIndexName("bar");
         Client client = clientBuilder.build();
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         ImmutableOpenMap<String, IndexMetaData> indexMap = ImmutableOpenMap.<String, IndexMetaData>builder().build();
 
@@ -248,7 +248,7 @@ public void testBuckets_OneBucketNoInterim() throws IOException {
         int from = 0;
         int size = 10;
         Client client = getMockedClient(queryBuilder -> queryBuilderHolder[0] = queryBuilder, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         BucketsQueryBuilder bq = new BucketsQueryBuilder().from(from).size(size).anomalyScoreThreshold(1.0);
 
@@ -281,7 +281,7 @@ public void testBuckets_OneBucketInterim() throws IOException {
         int size = 17;
 
         Client client = getMockedClient(queryBuilder -> queryBuilderHolder[0] = queryBuilder, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         BucketsQueryBuilder bq = new BucketsQueryBuilder().from(from).size(size).anomalyScoreThreshold(5.1)
                 .includeInterim(true);
@@ -314,7 +314,7 @@ public void testBuckets_UsingBuilder() throws IOException {
         int size = 17;
 
         Client client = getMockedClient(queryBuilder -> queryBuilderHolder[0] = queryBuilder, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         BucketsQueryBuilder bq = new BucketsQueryBuilder();
         bq.from(from);
@@ -341,7 +341,7 @@ public void testBucket_NoBucketNoExpand() throws IOException {
         SearchResponse response = createSearchResponse(source);
 
         Client client = getMockedClient(queryBuilder -> {}, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         BucketsQueryBuilder bq = new BucketsQueryBuilder();
         bq.timestamp(Long.toString(timestamp));
@@ -363,7 +363,7 @@ public void testBucket_OneBucketNoExpand() throws IOException {
 
         SearchResponse response = createSearchResponse(source);
         Client client = getMockedClient(queryBuilder -> {}, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         BucketsQueryBuilder bq = new BucketsQueryBuilder();
         bq.timestamp(Long.toString(now.getTime()));
@@ -403,7 +403,7 @@ public void testRecords() throws IOException {
         String sortfield = "minefield";
         SearchResponse response = createSearchResponse(source);
         Client client = getMockedClient(qb -> {}, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         RecordsQueryBuilder rqb = new RecordsQueryBuilder().from(from).size(size).epochStart(String.valueOf(now.getTime()))
                 .epochEnd(String.valueOf(now.getTime())).includeInterim(true).sortField(sortfield)
@@ -451,7 +451,7 @@ public void testRecords_UsingBuilder() throws IOException {
         SearchResponse response = createSearchResponse(source);
 
         Client client = getMockedClient(qb -> {}, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         RecordsQueryBuilder rqb = new RecordsQueryBuilder();
         rqb.from(from);
@@ -505,7 +505,7 @@ public void testBucketRecords() throws IOException {
         String sortfield = "minefield";
         SearchResponse response = createSearchResponse(source);
         Client client = getMockedClient(qb -> {}, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         @SuppressWarnings({"unchecked"})
         QueryPage<AnomalyRecord>[] holder = new QueryPage[1];
@@ -542,7 +542,7 @@ public void testexpandBucket() throws IOException {
 
         SearchResponse response = createSearchResponse(source);
         Client client = getMockedClient(qb -> {}, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         Integer[] holder = new Integer[1];
         provider.expandBucket(jobId, false, bucket, records -> holder[0] = records, RuntimeException::new, client);
@@ -567,7 +567,7 @@ public void testCategoryDefinitions() throws IOException {
         int size = 10;
         Client client = getMockedClient(q -> {}, response);
 
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
         @SuppressWarnings({"unchecked"})
         QueryPage<CategoryDefinition>[] holder = new QueryPage[1];
         provider.categoryDefinitions(jobId, null, false, from, size, r -> holder[0] = r,
@@ -589,7 +589,7 @@ public void testCategoryDefinition() throws IOException {
 
         SearchResponse response = createSearchResponse(Collections.singletonList(source));
         Client client = getMockedClient(q -> {}, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
         @SuppressWarnings({"unchecked"})
         QueryPage<CategoryDefinition>[] holder = new QueryPage[1];
         provider.categoryDefinitions(jobId, categoryId, false, null, null,
@@ -630,7 +630,7 @@ public void testInfluencers_NoInterim() throws IOException {
         QueryBuilder[] qbHolder = new QueryBuilder[1];
         SearchResponse response = createSearchResponse(source);
         Client client = getMockedClient(q -> qbHolder[0] = q, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         @SuppressWarnings({"unchecked"})
         QueryPage<Influencer>[] holder = new QueryPage[1];
@@ -690,7 +690,7 @@ public void testInfluencers_WithInterim() throws IOException {
         QueryBuilder[] qbHolder = new QueryBuilder[1];
         SearchResponse response = createSearchResponse(source);
         Client client = getMockedClient(q -> qbHolder[0] = q, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         @SuppressWarnings({"unchecked"})
         QueryPage<Influencer>[] holder = new QueryPage[1];
@@ -745,7 +745,7 @@ public void testModelSnapshots() throws IOException {
         int size = 3;
         SearchResponse response = createSearchResponse(source);
         Client client = getMockedClient(qb -> {}, response);
-        JobProvider provider = createProvider(client);
+        JobResultsProvider provider = createProvider(client);
 
         @SuppressWarnings({"unchecked"})
         QueryPage<ModelSnapshot>[] holder = new QueryPage[1];
@@ -783,11 +783,11 @@ public void testViolatedFieldCountLimit() throws Exception {
         MetaData metaData = MetaData.builder()
                 .put(indexMetaData1)
                 .build();
-        boolean result = JobProvider.violatedFieldCountLimit("index1", 0, 10,
+        boolean result = JobResultsProvider.violatedFieldCountLimit("index1", 0, 10,
                 ClusterState.builder(new ClusterName("_name")).metaData(metaData).build());
         assertFalse(result);
 
-        result = JobProvider.violatedFieldCountLimit("index1", 1, 10,
+        result = JobResultsProvider.violatedFieldCountLimit("index1", 1, 10,
                 ClusterState.builder(new ClusterName("_name")).metaData(metaData).build());
         assertTrue(result);
 
@@ -801,7 +801,7 @@ public void testViolatedFieldCountLimit() throws Exception {
         metaData = MetaData.builder()
                 .put(indexMetaData2)
                 .build();
-        result = JobProvider.violatedFieldCountLimit("index1", 0, 19,
+        result = JobResultsProvider.violatedFieldCountLimit("index1", 0, 19,
                 ClusterState.builder(new ClusterName("_name")).metaData(metaData).build());
         assertTrue(result);
     }
@@ -811,7 +811,7 @@ public void testCountFields() {
         mapping.put("field1", Collections.singletonMap("type", "string"));
         mapping.put("field2", Collections.singletonMap("type", "string"));
         mapping.put("field3", Collections.singletonMap("type", "string"));
-        assertEquals(3, JobProvider.countFields(Collections.singletonMap("properties", mapping)));
+        assertEquals(3, JobResultsProvider.countFields(Collections.singletonMap("properties", mapping)));
 
         Map<String, Object> objectProperties = new HashMap<>();
         objectProperties.put("field4", Collections.singletonMap("type", "string"));
@@ -822,15 +822,15 @@ public void testCountFields() {
         objectField.put("properties", objectProperties);
 
         mapping.put("field4", objectField);
-        assertEquals(7, JobProvider.countFields(Collections.singletonMap("properties", mapping)));
+        assertEquals(7, JobResultsProvider.countFields(Collections.singletonMap("properties", mapping)));
     }
 
     private Bucket createBucketAtEpochTime(long epoch) {
         return new Bucket("foo", new Date(epoch), 123);
     }
 
-    private JobProvider createProvider(Client client) {
-        return new JobProvider(client, Settings.EMPTY);
+    private JobResultsProvider createProvider(Client client) {
+        return new JobResultsProvider(client, Settings.EMPTY);
     }
 
     private static GetResponse createGetResponse(boolean exists, Map<String, Object> source) throws IOException {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrlTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrlTests.java
deleted file mode 100644
index 8091c2a02c371..0000000000000
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrlTests.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml.job.process;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.TestEnvironment;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
-import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
-import org.elasticsearch.xpack.core.ml.job.config.Detector;
-import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.mockito.Mockito;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder;
-
-public class ProcessCtrlTests extends ESTestCase {
-
-    private final Logger logger = Mockito.mock(Logger.class);
-
-    public void testBuildAutodetectCommand() {
-        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
-        Environment env = TestEnvironment.newEnvironment(settings);
-        Job.Builder job = buildJobBuilder("unit-test-job");
-
-        Detector.Builder detectorBuilder = new Detector.Builder("mean", "value");
-        detectorBuilder.setPartitionFieldName("foo");
-        AnalysisConfig.Builder acBuilder = new AnalysisConfig.Builder(Collections.singletonList(detectorBuilder.build()));
-        acBuilder.setBucketSpan(TimeValue.timeValueSeconds(120));
-        acBuilder.setLatency(TimeValue.timeValueSeconds(360));
-        acBuilder.setSummaryCountFieldName("summaryField");
-        acBuilder.setOverlappingBuckets(true);
-        acBuilder.setMultivariateByFields(true);
-        acBuilder.setUsePerPartitionNormalization(true);
-        job.setAnalysisConfig(acBuilder);
-
-        DataDescription.Builder dd = new DataDescription.Builder();
-        dd.setFormat(DataDescription.DataFormat.DELIMITED);
-        dd.setFieldDelimiter('|');
-        dd.setTimeField("tf");
-        job.setDataDescription(dd);
-
-        List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger);
-        assertEquals(13, command.size());
-        assertTrue(command.contains(ProcessCtrl.AUTODETECT_PATH));
-        assertTrue(command.contains(ProcessCtrl.BUCKET_SPAN_ARG + "120"));
-        assertTrue(command.contains(ProcessCtrl.LATENCY_ARG + "360"));
-        assertTrue(command.contains(ProcessCtrl.SUMMARY_COUNT_FIELD_ARG + "summaryField"));
-        assertTrue(command.contains(ProcessCtrl.RESULT_FINALIZATION_WINDOW_ARG + "2"));
-        assertTrue(command.contains(ProcessCtrl.MULTIVARIATE_BY_FIELDS_ARG));
-
-        assertTrue(command.contains(ProcessCtrl.LENGTH_ENCODED_INPUT_ARG));
-        assertTrue(command.contains(ProcessCtrl.maxAnomalyRecordsArg(settings)));
-
-        assertTrue(command.contains(ProcessCtrl.TIME_FIELD_ARG + "tf"));
-        assertTrue(command.contains(ProcessCtrl.JOB_ID_ARG + "unit-test-job"));
-
-        assertTrue(command.contains(ProcessCtrl.PER_PARTITION_NORMALIZATION));
-
-        int expectedPersistInterval = 10800 + ProcessCtrl.calculateStaggeringInterval(job.getId());
-        assertTrue(command.contains(ProcessCtrl.PERSIST_INTERVAL_ARG + expectedPersistInterval));
-        int expectedMaxQuantileInterval = 21600 + ProcessCtrl.calculateStaggeringInterval(job.getId());
-        assertTrue(command.contains(ProcessCtrl.MAX_QUANTILE_INTERVAL_ARG + expectedMaxQuantileInterval));
-    }
-
-    public void testBuildAutodetectCommand_defaultTimeField() {
-        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
-        Environment env = TestEnvironment.newEnvironment(settings);
-        Job.Builder job = buildJobBuilder("unit-test-job");
-
-        List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger);
-
-        assertTrue(command.contains(ProcessCtrl.TIME_FIELD_ARG + "time"));
-    }
-
-    public void testBuildAutodetectCommand_givenPersistModelState() {
-        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put(ProcessCtrl.DONT_PERSIST_MODEL_STATE_SETTING.getKey(), true).build();
-        Environment env = TestEnvironment.newEnvironment(settings);
-        Job.Builder job = buildJobBuilder("unit-test-job");
-
-        int expectedPersistInterval = 10800 + ProcessCtrl.calculateStaggeringInterval(job.getId());
-
-        List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger);
-        assertFalse(command.contains(ProcessCtrl.PERSIST_INTERVAL_ARG + expectedPersistInterval));
-
-        settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
-        env = TestEnvironment.newEnvironment(settings);
-
-        command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger);
-        assertTrue(command.contains(ProcessCtrl.PERSIST_INTERVAL_ARG + expectedPersistInterval));
-    }
-
-    public void testBuildNormalizerCommand() throws IOException {
-        Environment env = TestEnvironment.newEnvironment(
-                Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build());
-        String jobId = "unit-test-job";
-
-        List<String> command = ProcessCtrl.buildNormalizerCommand(env, jobId, null, 300, true);
-        assertEquals(4, command.size());
-        assertTrue(command.contains(ProcessCtrl.NORMALIZE_PATH));
-        assertTrue(command.contains(ProcessCtrl.BUCKET_SPAN_ARG + "300"));
-        assertTrue(command.contains(ProcessCtrl.LENGTH_ENCODED_INPUT_ARG));
-        assertTrue(command.contains(ProcessCtrl.PER_PARTITION_NORMALIZATION));
-    }
-}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/ProcessPipesTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/ProcessPipesTests.java
index 37aeadaafde56..708d7af152014 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/ProcessPipesTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/ProcessPipesTests.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectBuilder;
 import org.elasticsearch.xpack.ml.utils.NamedPipeHelper;
 import org.mockito.Mockito;
 
@@ -50,7 +51,7 @@ public void testProcessPipes() throws IOException {
         when(namedPipeHelper.openNamedPipeInputStream(contains("persist"), any(Duration.class)))
                 .thenReturn(new ByteArrayInputStream(PERSIST_BYTES));
 
-        ProcessPipes processPipes = new ProcessPipes(env, namedPipeHelper, ProcessCtrl.AUTODETECT, "my_job",
+        ProcessPipes processPipes = new ProcessPipes(env, namedPipeHelper, AutodetectBuilder.AUTODETECT, "my_job",
                 true, false, true, true, true, true);
 
         List<String> command = new ArrayList<>();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java
new file mode 100644
index 0000000000000..0f83106441185
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job.process.autodetect;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
+import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
+import org.elasticsearch.xpack.core.ml.job.config.Detector;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.ml.job.process.NativeController;
+import org.elasticsearch.xpack.ml.job.process.ProcessPipes;
+import org.junit.Before;
+
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder;
+import static org.mockito.Mockito.mock;
+
+public class AutodetectBuilderTests extends ESTestCase {
+
+    private Logger logger;
+    private List<Path> filesToDelete;
+    private Environment env;
+    private Settings settings;
+    private NativeController nativeController;
+    private ProcessPipes processPipes;
+
+    @Before
+    public void setUpTests() {
+        logger = mock(Logger.class);
+        filesToDelete = Collections.emptyList();
+        settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
+        env = TestEnvironment.newEnvironment(settings);
+        nativeController = mock(NativeController.class);
+        processPipes = mock(ProcessPipes.class);
+    }
+
+    public void testBuildAutodetectCommand() {
+        Job.Builder job = buildJobBuilder("unit-test-job");
+
+        Detector.Builder detectorBuilder = new Detector.Builder("mean", "value");
+        detectorBuilder.setPartitionFieldName("foo");
+        AnalysisConfig.Builder acBuilder = new AnalysisConfig.Builder(Collections.singletonList(detectorBuilder.build()));
+        acBuilder.setBucketSpan(TimeValue.timeValueSeconds(120));
+        acBuilder.setLatency(TimeValue.timeValueSeconds(360));
+        acBuilder.setSummaryCountFieldName("summaryField");
+        acBuilder.setOverlappingBuckets(true);
+        acBuilder.setMultivariateByFields(true);
+        acBuilder.setUsePerPartitionNormalization(true);
+        job.setAnalysisConfig(acBuilder);
+
+        DataDescription.Builder dd = new DataDescription.Builder();
+        dd.setFormat(DataDescription.DataFormat.DELIMITED);
+        dd.setFieldDelimiter('|');
+        dd.setTimeField("tf");
+        job.setDataDescription(dd);
+
+        List<String> command = autodetectBuilder(job.build()).buildAutodetectCommand();
+        assertEquals(13, command.size());
+        assertTrue(command.contains(AutodetectBuilder.AUTODETECT_PATH));
+        assertTrue(command.contains(AutodetectBuilder.BUCKET_SPAN_ARG + "120"));
+        assertTrue(command.contains(AutodetectBuilder.LATENCY_ARG + "360"));
+        assertTrue(command.contains(AutodetectBuilder.SUMMARY_COUNT_FIELD_ARG + "summaryField"));
+        assertTrue(command.contains(AutodetectBuilder.RESULT_FINALIZATION_WINDOW_ARG + "2"));
+        assertTrue(command.contains(AutodetectBuilder.MULTIVARIATE_BY_FIELDS_ARG));
+
+        assertTrue(command.contains(AutodetectBuilder.LENGTH_ENCODED_INPUT_ARG));
+        assertTrue(command.contains(AutodetectBuilder.maxAnomalyRecordsArg(settings)));
+
+        assertTrue(command.contains(AutodetectBuilder.TIME_FIELD_ARG + "tf"));
+        assertTrue(command.contains(AutodetectBuilder.JOB_ID_ARG + "unit-test-job"));
+
+        assertTrue(command.contains(AutodetectBuilder.PER_PARTITION_NORMALIZATION));
+
+        int expectedPersistInterval = 10800 + AutodetectBuilder.calculateStaggeringInterval(job.getId());
+        assertTrue(command.contains(AutodetectBuilder.PERSIST_INTERVAL_ARG + expectedPersistInterval));
+        int expectedMaxQuantileInterval = 21600 + AutodetectBuilder.calculateStaggeringInterval(job.getId());
+        assertTrue(command.contains(AutodetectBuilder.MAX_QUANTILE_INTERVAL_ARG + expectedMaxQuantileInterval));
+    }
+
+    public void testBuildAutodetectCommand_defaultTimeField() {
+        Job.Builder job = buildJobBuilder("unit-test-job");
+
+        List<String> command = autodetectBuilder(job.build()).buildAutodetectCommand();
+
+        assertTrue(command.contains(AutodetectBuilder.TIME_FIELD_ARG + "time"));
+    }
+
+    public void testBuildAutodetectCommand_givenPersistModelState() {
+        settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put(AutodetectBuilder.DONT_PERSIST_MODEL_STATE_SETTING.getKey(), true).build();
+        Job.Builder job = buildJobBuilder("unit-test-job");
+
+        int expectedPersistInterval = 10800 + AutodetectBuilder.calculateStaggeringInterval(job.getId());
+
+        List<String> command = autodetectBuilder(job.build()).buildAutodetectCommand();
+        assertFalse(command.contains(AutodetectBuilder.PERSIST_INTERVAL_ARG + expectedPersistInterval));
+
+        settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
+        env = TestEnvironment.newEnvironment(settings);
+
+        command = autodetectBuilder(job.build()).buildAutodetectCommand();
+        assertTrue(command.contains(AutodetectBuilder.PERSIST_INTERVAL_ARG + expectedPersistInterval));
+    }
+
+    private AutodetectBuilder autodetectBuilder(Job job) {
+        return new AutodetectBuilder(job, filesToDelete, logger, env, settings, nativeController, processPipes);
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java
index ab24aadb9dc3a..fda96ca29a695 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java
@@ -108,7 +108,7 @@ public void testWriteUpdateProcessMessage() throws IOException {
         verifyNoMoreInteractions(process);
     }
 
-    public void testFlushJob() throws IOException {
+    public void testFlushJob() throws IOException, InterruptedException {
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
         when(process.isProcessAlive()).thenReturn(true);
         AutoDetectResultProcessor processor = mock(AutoDetectResultProcessor.class);
@@ -123,7 +123,7 @@ public void testFlushJob() throws IOException {
         }
     }
 
-    public void testWaitForFlushReturnsIfParserFails() throws IOException {
+    public void testWaitForFlushReturnsIfParserFails() throws IOException, InterruptedException {
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
         when(process.isProcessAlive()).thenReturn(true);
         AutoDetectResultProcessor processor = mock(AutoDetectResultProcessor.class);
@@ -144,7 +144,7 @@ public void testFlushJob_throwsIfProcessIsDead() throws IOException {
         assertEquals("[foo] Unexpected death of autodetect: Mock process is dead", holder[0].getMessage());
     }
 
-    public void testFlushJob_givenFlushWaitReturnsTrueOnSecondCall() throws IOException {
+    public void testFlushJob_givenFlushWaitReturnsTrueOnSecondCall() throws IOException, InterruptedException {
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
         when(process.isProcessAlive()).thenReturn(true);
         AutoDetectResultProcessor autoDetectResultProcessor = Mockito.mock(AutoDetectResultProcessor.class);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
index 313f449cadd81..43cc909e392ea 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
@@ -39,7 +39,7 @@
 import org.elasticsearch.xpack.ml.job.JobManager;
 import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzerTests;
 import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams;
@@ -101,7 +101,7 @@ public class AutodetectProcessManagerTests extends ESTestCase {
     private Environment environment;
     private AnalysisRegistry analysisRegistry;
     private JobManager jobManager;
-    private JobProvider jobProvider;
+    private JobResultsProvider jobResultsProvider;
     private JobResultsPersister jobResultsPersister;
     private JobDataCountsPersister jobDataCountsPersister;
     private NormalizerFactory normalizerFactory;
@@ -119,7 +119,7 @@ public void setup() throws Exception {
         environment = TestEnvironment.newEnvironment(settings);
         analysisRegistry = CategorizationAnalyzerTests.buildTestAnalysisRegistry(environment);
         jobManager = mock(JobManager.class);
-        jobProvider = mock(JobProvider.class);
+        jobResultsProvider = mock(JobResultsProvider.class);
         jobResultsPersister = mock(JobResultsPersister.class);
         when(jobResultsPersister.bulkPersisterBuilder(any())).thenReturn(mock(JobResultsPersister.Builder.class));
         jobDataCountsPersister = mock(JobDataCountsPersister.class);
@@ -132,7 +132,7 @@ public void setup() throws Exception {
             Consumer<AutodetectParams> handler = (Consumer<AutodetectParams>) invocationOnMock.getArguments()[1];
             handler.accept(buildAutodetectParams());
             return null;
-        }).when(jobProvider).getAutodetectParams(any(), any(), any());
+        }).when(jobResultsProvider).getAutodetectParams(any(), any(), any());
     }
 
     public void testMaxOpenJobsSetting_givenDefault() {
@@ -227,7 +227,7 @@ public void testOpenJob_exceedMaxNumJobs() {
         Settings.Builder settings = Settings.builder();
         settings.put(AutodetectProcessManager.MAX_OPEN_JOBS_PER_NODE.getKey(), 3);
         AutodetectProcessManager manager = spy(new AutodetectProcessManager(environment, settings.build(), client, threadPool,
-                jobManager, jobProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory,
+                jobManager, jobResultsProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory,
                 normalizerFactory, new NamedXContentRegistry(Collections.emptyList()), auditor));
         doReturn(executorService).when(manager).createAutodetectExecutorService(any());
 
@@ -582,7 +582,7 @@ public void testCreate_notEnoughThreads() throws IOException {
         AutodetectProcessFactory autodetectProcessFactory =
                 (j, autodetectParams, e, onProcessCrash) -> autodetectProcess;
         AutodetectProcessManager manager = new AutodetectProcessManager(environment, Settings.EMPTY,
-                client, threadPool, jobManager, jobProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory,
+                client, threadPool, jobManager, jobResultsProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory,
                 normalizerFactory, new NamedXContentRegistry(Collections.emptyList()), auditor);
 
         JobTask jobTask = mock(JobTask.class);
@@ -655,7 +655,7 @@ private AutodetectProcessManager createNonSpyManager(String jobId) {
         AutodetectProcessFactory autodetectProcessFactory =
                 (j, autodetectParams, e, onProcessCrash) -> autodetectProcess;
         return new AutodetectProcessManager(environment, Settings.EMPTY, client, threadPool, jobManager,
-                jobProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory,
+                jobResultsProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory,
                 normalizerFactory, new NamedXContentRegistry(Collections.emptyList()), auditor);
     }
 
@@ -680,7 +680,7 @@ private AutodetectProcessManager createManager(AutodetectCommunicator communicat
         when(threadPool.executor(anyString())).thenReturn(EsExecutors.newDirectExecutorService());
         AutodetectProcessFactory autodetectProcessFactory = mock(AutodetectProcessFactory.class);
         AutodetectProcessManager manager = new AutodetectProcessManager(environment, Settings.EMPTY,
-                client, threadPool, jobManager, jobProvider, jobResultsPersister, jobDataCountsPersister,
+                client, threadPool, jobManager, jobResultsProvider, jobResultsPersister, jobDataCountsPersister,
                 autodetectProcessFactory, normalizerFactory,
                 new NamedXContentRegistry(Collections.emptyList()), auditor);
         manager = spy(manager);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java
index 8eb0317ba0dbe..c5a459f70f3f2 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
 import org.elasticsearch.xpack.core.ml.job.results.Influencer;
 import org.elasticsearch.xpack.core.ml.job.results.ModelPlot;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess;
 import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer;
@@ -75,7 +75,7 @@ public class AutoDetectResultProcessorTests extends ESTestCase {
     private Auditor auditor;
     private Renormalizer renormalizer;
     private JobResultsPersister persister;
-    private JobProvider jobProvider;
+    private JobResultsProvider jobResultsProvider;
     private FlushListener flushListener;
     private AutoDetectResultProcessor processorUnderTest;
     private ScheduledThreadPoolExecutor executor;
@@ -90,9 +90,9 @@ public void setUpMocks() {
         when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
         renormalizer = mock(Renormalizer.class);
         persister = mock(JobResultsPersister.class);
-        jobProvider = mock(JobProvider.class);
+        jobResultsProvider = mock(JobResultsProvider.class);
         flushListener = mock(FlushListener.class);
-        processorUnderTest = new AutoDetectResultProcessor(client, auditor, JOB_ID, renormalizer, persister, jobProvider,
+        processorUnderTest = new AutoDetectResultProcessor(client, auditor, JOB_ID, renormalizer, persister, jobResultsProvider,
                 new ModelSizeStats.Builder(JOB_ID).setTimestamp(new Date(BUCKET_SPAN_MS)).build(), false, flushListener);
     }
 
@@ -294,8 +294,8 @@ public void testProcessResult_modelSizeStats() {
 
         verify(persister, times(1)).persistModelSizeStats(modelSizeStats);
         verifyNoMoreInteractions(persister);
-        // No interactions with the jobProvider confirms that the established memory calculation did not run
-        verifyNoMoreInteractions(jobProvider, auditor);
+        // No interactions with the jobResultsProvider confirms that the established memory calculation did not run
+        verifyNoMoreInteractions(jobResultsProvider, auditor);
         assertEquals(modelSizeStats, processorUnderTest.modelSizeStats());
     }
 
@@ -347,7 +347,7 @@ public void testProcessResult_modelSizeStatsAfterManyBuckets() throws Exception
 
         AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
         context.deleteInterimRequired = false;
-        for (int i = 0; i < JobProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE; ++i) {
+        for (int i = 0; i < JobResultsProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE; ++i) {
             AutodetectResult result = mock(AutodetectResult.class);
             Bucket bucket = mock(Bucket.class);
             when(result.getBucket()).thenReturn(bucket);
@@ -366,9 +366,9 @@ public void testProcessResult_modelSizeStatsAfterManyBuckets() throws Exception
             verify(persister, times(1)).persistModelSizeStats(modelSizeStats);
             verify(persister, times(1)).commitResultWrites(JOB_ID);
             verifyNoMoreInteractions(persister);
-            verify(jobProvider, times(1)).getEstablishedMemoryUsage(eq(JOB_ID), eq(timestamp), eq(modelSizeStats), any(Consumer.class),
-                any(Consumer.class));
-            verifyNoMoreInteractions(jobProvider);
+            verify(jobResultsProvider, times(1)).getEstablishedMemoryUsage(eq(JOB_ID), eq(timestamp),
+                    eq(modelSizeStats), any(Consumer.class), any(Consumer.class));
+            verifyNoMoreInteractions(jobResultsProvider);
             assertEquals(modelSizeStats, processorUnderTest.modelSizeStats());
         });
     }
@@ -383,13 +383,13 @@ public void testProcessResult_manyModelSizeStatsInQuickSuccession() throws Excep
         AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
         context.deleteInterimRequired = false;
         ModelSizeStats modelSizeStats = null;
-        for (int i = 1; i <= JobProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE + 5; ++i) {
+        for (int i = 1; i <= JobResultsProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE + 5; ++i) {
             AutodetectResult result = mock(AutodetectResult.class);
             Bucket bucket = mock(Bucket.class);
             when(bucket.getTimestamp()).thenReturn(new Date(BUCKET_SPAN_MS * i));
             when(result.getBucket()).thenReturn(bucket);
             processorUnderTest.processResult(context, result);
-            if (i > JobProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE) {
+            if (i > JobResultsProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE) {
                 result = mock(AutodetectResult.class);
                 modelSizeStats = mock(ModelSizeStats.class);
                 when(modelSizeStats.getTimestamp()).thenReturn(new Date(BUCKET_SPAN_MS * i));
@@ -409,9 +409,9 @@ public void testProcessResult_manyModelSizeStatsInQuickSuccession() throws Excep
             // ...but only the last should trigger an established model memory update
             verify(persister, times(1)).commitResultWrites(JOB_ID);
             verifyNoMoreInteractions(persister);
-            verify(jobProvider, times(1)).getEstablishedMemoryUsage(eq(JOB_ID), eq(lastTimestamp), eq(lastModelSizeStats),
+            verify(jobResultsProvider, times(1)).getEstablishedMemoryUsage(eq(JOB_ID), eq(lastTimestamp), eq(lastModelSizeStats),
                 any(Consumer.class), any(Consumer.class));
-            verifyNoMoreInteractions(jobProvider);
+            verifyNoMoreInteractions(jobResultsProvider);
             assertEquals(lastModelSizeStats, processorUnderTest.modelSizeStats());
         });
     }
@@ -514,7 +514,7 @@ public void testPersisterThrowingDoesntBlockProcessing() {
         verify(persister, times(2)).persistModelSnapshot(any(), eq(WriteRequest.RefreshPolicy.IMMEDIATE));
     }
 
-    public void testParsingErrorSetsFailed() {
+    public void testParsingErrorSetsFailed() throws InterruptedException {
         @SuppressWarnings("unchecked")
         Iterator<AutodetectResult> iterator = mock(Iterator.class);
         when(iterator.hasNext()).thenThrow(new ElasticsearchParseException("this test throws"));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushListenerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushListenerTests.java
index 70ba757f148fe..3bcedb523923e 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushListenerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushListenerTests.java
@@ -22,8 +22,12 @@ public void testAcknowledgeFlush() throws Exception {
         FlushListener listener = new FlushListener();
         AtomicReference<FlushAcknowledgement> flushAcknowledgementHolder = new AtomicReference<>();
         new Thread(() -> {
-            FlushAcknowledgement flushAcknowledgement = listener.waitForFlush("_id", Duration.ofMillis(10000));
-            flushAcknowledgementHolder.set(flushAcknowledgement);
+            try {
+                FlushAcknowledgement flushAcknowledgement = listener.waitForFlush("_id", Duration.ofMillis(10000));
+                flushAcknowledgementHolder.set(flushAcknowledgement);
+            } catch (InterruptedException _ex) {
+                Thread.currentThread().interrupt();
+            }
         }).start();
         assertBusy(() -> assertTrue(listener.awaitingFlushed.containsKey("_id")));
         assertNull(flushAcknowledgementHolder.get());
@@ -46,8 +50,12 @@ public void testClear() throws Exception {
             AtomicReference<FlushAcknowledgement> flushAcknowledgementHolder = new AtomicReference<>();
             flushAcknowledgementHolders.add(flushAcknowledgementHolder);
             new Thread(() -> {
-                FlushAcknowledgement flushAcknowledgement = listener.waitForFlush(String.valueOf(id), Duration.ofMillis(10000));
-                flushAcknowledgementHolder.set(flushAcknowledgement);
+                try {
+                    FlushAcknowledgement flushAcknowledgement = listener.waitForFlush(String.valueOf(id), Duration.ofMillis(10000));
+                    flushAcknowledgementHolder.set(flushAcknowledgement);
+                } catch (InterruptedException _ex) {
+                    Thread.currentThread().interrupt();
+                }
             }).start();
         }
         assertBusy(() -> assertEquals(numWaits, listener.awaitingFlushed.size()));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketInfluencerNormalizableTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketInfluencerNormalizableTests.java
index f83d51d84b009..bde5c3e44f9b4 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketInfluencerNormalizableTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketInfluencerNormalizableTests.java
@@ -43,10 +43,18 @@ public void testGetPartitionFieldName() {
         assertNull(new BucketInfluencerNormalizable(bucketInfluencer, INDEX_NAME).getPartitionFieldName());
     }
 
+    public void testGetPartitionFieldValue() {
+        assertNull(new BucketInfluencerNormalizable(bucketInfluencer, INDEX_NAME).getPartitionFieldValue());
+    }
+
     public void testGetPersonFieldName() {
         assertEquals("airline", new BucketInfluencerNormalizable(bucketInfluencer, INDEX_NAME).getPersonFieldName());
     }
 
+    public void testGetPersonFieldValue() {
+        assertNull(new BucketInfluencerNormalizable(bucketInfluencer, INDEX_NAME).getPersonFieldValue());
+    }
+
     public void testGetFunctionName() {
         assertNull(new BucketInfluencerNormalizable(bucketInfluencer, INDEX_NAME).getFunctionName());
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java
index 630bffe11129a..4436fcc7026fe 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java
@@ -73,6 +73,10 @@ public void testGetPersonFieldName() {
         assertNull(new BucketNormalizable(bucket, INDEX_NAME).getPersonFieldName());
     }
 
+    public void testGetPersonFieldValue() {
+        assertNull(new BucketNormalizable(bucket, INDEX_NAME).getPersonFieldValue());
+    }
+
     public void testGetFunctionName() {
         assertNull(new BucketNormalizable(bucket, INDEX_NAME).getFunctionName());
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizableTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizableTests.java
index 215f88ad33224..ee5518b9c12d5 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizableTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizableTests.java
@@ -44,6 +44,10 @@ public void testGetPersonFieldName() {
         assertEquals("airline", new InfluencerNormalizable(influencer, INDEX_NAME).getPersonFieldName());
     }
 
+    public void testGetPersonFieldValue() {
+        assertEquals("AAL", new InfluencerNormalizable(influencer, INDEX_NAME).getPersonFieldValue());
+    }
+
     public void testGetFunctionName() {
         assertNull(new InfluencerNormalizable(influencer, INDEX_NAME).getFunctionName());
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilderTests.java
new file mode 100644
index 0000000000000..64e595fd5a043
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilderTests.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job.process.normalizer;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectBuilder;
+
+import java.io.IOException;
+import java.util.List;
+
+public class NormalizerBuilderTests extends ESTestCase {
+
+    public void testBuildNormalizerCommand() throws IOException {
+        Environment env = TestEnvironment.newEnvironment(
+                Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build());
+        String jobId = "unit-test-job";
+
+        List<String> command = new NormalizerBuilder(env, jobId, null, 300, true).build();
+        assertEquals(4, command.size());
+        assertTrue(command.contains("./normalize"));
+        assertTrue(command.contains(AutodetectBuilder.BUCKET_SPAN_ARG + "300"));
+        assertTrue(command.contains(AutodetectBuilder.LENGTH_ENCODED_INPUT_ARG));
+        assertTrue(command.contains(AutodetectBuilder.PER_PARTITION_NORMALIZATION));
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResultTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResultTests.java
index ecaea449f95c1..af35c01aa871d 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResultTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResultTests.java
@@ -19,6 +19,7 @@ public void testDefaultConstructor() {
         assertNull(msg.getPartitionFieldName());
         assertNull(msg.getPartitionFieldValue());
         assertNull(msg.getPersonFieldName());
+        assertNull(msg.getPersonFieldValue());
         assertNull(msg.getFunctionName());
         assertNull(msg.getValueFieldName());
         assertEquals(0.0, msg.getProbability(), EPSILON);
@@ -32,6 +33,7 @@ protected NormalizerResult createTestInstance() {
         msg.setPartitionFieldName("part");
         msg.setPartitionFieldValue("something");
         msg.setPersonFieldName("person");
+        msg.setPersonFieldValue("fred");
         msg.setFunctionName("mean");
         msg.setValueFieldName("value");
         msg.setProbability(0.005);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java
index 5f8b685f8442c..2acaf97359477 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.xpack.core.ml.job.results.BucketInfluencer;
 import org.elasticsearch.xpack.core.ml.job.results.Influencer;
 import org.elasticsearch.xpack.core.ml.job.results.Result;
-import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobRenormalizedResultsPersister;
 import org.elasticsearch.xpack.ml.job.persistence.MockBatchedDocumentsIterator;
 import org.junit.Before;
@@ -52,7 +52,7 @@ public class ScoresUpdaterTests extends ESTestCase {
     private static final long DEFAULT_BUCKET_SPAN = 3600;
     private static final long DEFAULT_START_TIME = 0;
 
-    private JobProvider jobProvider = mock(JobProvider.class);
+    private JobResultsProvider jobResultsProvider = mock(JobResultsProvider.class);
     private JobRenormalizedResultsPersister jobRenormalizedResultsPersister = mock(JobRenormalizedResultsPersister.class);
     private Normalizer normalizer = mock(Normalizer.class);
     private NormalizerFactory normalizerFactory = mock(NormalizerFactory.class);
@@ -78,7 +78,7 @@ public void setUpMocks() throws IOException {
 
         job = jobBuilder.build(new Date());
 
-        scoresUpdater = new ScoresUpdater(job, jobProvider, jobRenormalizedResultsPersister, normalizerFactory);
+        scoresUpdater = new ScoresUpdater(job, jobResultsProvider, jobRenormalizedResultsPersister, normalizerFactory);
 
         givenProviderReturnsNoBuckets();
         givenProviderReturnsNoRecords();
@@ -210,7 +210,7 @@ public void testUpdate_GivenTwoBucketsWithFirstHavingEnoughRecordsToForceSecondN
         MockBatchedDocumentsIterator<AnomalyRecord> recordIter = new MockBatchedDocumentsIterator<>(
                 recordBatches, AnomalyRecord.RESULT_TYPE_VALUE);
         recordIter.requireIncludeInterim(false);
-        when(jobProvider.newBatchedRecordsIterator(JOB_ID)).thenReturn(recordIter);
+        when(jobResultsProvider.newBatchedRecordsIterator(JOB_ID)).thenReturn(recordIter);
 
         scoresUpdater.update(QUANTILES_STATE, 3600, 0, false);
 
@@ -376,7 +376,7 @@ private void givenBuckets(List<Deque<Bucket>> batches) {
 
         MockBatchedDocumentsIterator<Bucket> bucketIter = new MockBatchedDocumentsIterator<>(batchesWithIndex, Bucket.RESULT_TYPE_VALUE);
         bucketIter.requireIncludeInterim(false);
-        when(jobProvider.newBatchedBucketsIterator(JOB_ID)).thenReturn(bucketIter);
+        when(jobResultsProvider.newBatchedBucketsIterator(JOB_ID)).thenReturn(bucketIter);
     }
 
     private void givenProviderReturnsNoRecords() {
@@ -394,7 +394,7 @@ private void givenProviderReturnsRecords(Deque<AnomalyRecord> records) {
         MockBatchedDocumentsIterator<AnomalyRecord> recordIter = new MockBatchedDocumentsIterator<>(
                 batches, AnomalyRecord.RESULT_TYPE_VALUE);
         recordIter.requireIncludeInterim(false);
-        when(jobProvider.newBatchedRecordsIterator(JOB_ID)).thenReturn(recordIter);
+        when(jobResultsProvider.newBatchedRecordsIterator(JOB_ID)).thenReturn(recordIter);
     }
 
     private void givenProviderReturnsNoInfluencers() {
@@ -410,7 +410,7 @@ private void givenProviderReturnsInfluencers(Deque<Influencer> influencers) {
         batches.add(queue);
         MockBatchedDocumentsIterator<Influencer> iterator = new MockBatchedDocumentsIterator<>(batches, Influencer.RESULT_TYPE_VALUE);
         iterator.requireIncludeInterim(false);
-        when(jobProvider.newBatchedInfluencersIterator(JOB_ID)).thenReturn(iterator);
+        when(jobResultsProvider.newBatchedInfluencersIterator(JOB_ID)).thenReturn(iterator);
     }
 
     private void verifyNormalizerWasInvoked(int times) throws IOException {
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
index 08618b45cf51f..d4375af4e553a 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
 import org.elasticsearch.action.ingest.PutPipelineRequest;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterChangedEvent;
@@ -39,6 +38,7 @@
 import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.PipelineConfiguration;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.xpack.core.XPackClient;
@@ -46,7 +46,6 @@
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
@@ -385,7 +384,7 @@ private boolean hasIngestPipeline(final ClusterState clusterState, final String
      * }
      * </code></pre>
      */
-    private void putIngestPipeline(final String pipelineId, final ActionListener<WritePipelineResponse> listener) {
+    private void putIngestPipeline(final String pipelineId, final ActionListener<AcknowledgedResponse> listener) {
         final String pipelineName = pipelineName(pipelineId);
         final BytesReference pipeline = BytesReference.bytes(loadPipeline(pipelineId, XContentType.JSON));
         final PutPipelineRequest request = new PutPipelineRequest(pipelineName, pipeline, XContentType.JSON);
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
index 098f4190b0e88..c7ddb3c4d2427 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
@@ -302,6 +302,7 @@ public void testToXContent() throws IOException {
         when(mockNodeResponse.shardsStats()).thenReturn(new ShardStats[]{mockShardStats});
 
         final ClusterStatsResponse clusterStats = new ClusterStatsResponse(1451606400000L,
+                                                                            "_cluster",
                                                                             clusterName,
                                                                             singletonList(mockNodeResponse),
                                                                             emptyList());
@@ -353,6 +354,7 @@ public void testToXContent() throws IOException {
                     + (needToEnableTLS ? ",\"cluster_needs_tls\":true" : "")
                   + "},"
                   + "\"cluster_stats\":{"
+                    + "\"cluster_uuid\":\"_cluster\","
                     + "\"timestamp\":1451606400000,"
                     + "\"status\":\"red\","
                     + "\"indices\":{"
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterSslIT.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterSslIT.java
index a0511dc17aa92..9b108d4f8c69f 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterSslIT.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterSslIT.java
@@ -144,7 +144,7 @@ public void testCanAddNewExporterWithSsl() {
             .put("xpack.monitoring.exporters._new.host", "https://" + webServer.getHostName() + ":" + webServer.getPort())
             .put("xpack.monitoring.exporters._new.ssl.truststore.path", truststore)
             .put("xpack.monitoring.exporters._new.ssl.truststore.password", "testnode")
-            .put("xpack.monitoring.exporters._new.ssl.verification_mode", VerificationMode.NONE.name())
+            .put("xpack.monitoring.exporters._new.ssl.verification_mode", VerificationMode.CERTIFICATE.name())
             .build();
         updateSettings.transientSettings(settings);
         final ActionFuture<ClusterUpdateSettingsResponse> future = client().admin().cluster().updateSettings(updateSettings);
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
index 546103df5dd46..0fc4d838f7ce8 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
@@ -80,7 +80,14 @@
 public class Rollup extends Plugin implements ActionPlugin, PersistentTaskPlugin {
 
     public static final String BASE_PATH = "/_xpack/rollup/";
-    public static final int ROLLUP_VERSION = 1;
+
+    // Introduced in ES version 6.3
+    public static final int ROLLUP_VERSION_V1 = 1;
+    // Introduced in ES Version 6.4
+    // Bumped due to ID collision, see #32372
+    public static final int ROLLUP_VERSION_V2 = 2;
+    public static final int CURRENT_ROLLUP_VERSION = ROLLUP_VERSION_V2;
+
     public static final String TASK_THREAD_POOL_NAME = RollupField.NAME + "_indexing";
     public static final String SCHEDULE_THREAD_POOL_NAME = RollupField.NAME + "_scheduler";
 
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtils.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtils.java
index da0ddadb972c4..d1706fd708e93 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtils.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtils.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.xpack.core.rollup.RollupField;
 import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
 import org.joda.time.DateTimeZone;
 
 import java.util.ArrayList;
@@ -94,7 +94,7 @@ private static void checkDateHisto(DateHistogramAggregationBuilder source, List<
                 for (Map<String, Object> agg : fieldCaps.getAggs()) {
                     if (agg.get(RollupField.AGG).equals(DateHistogramAggregationBuilder.NAME)) {
                         TimeValue interval = TimeValue.parseTimeValue((String)agg.get(RollupField.INTERVAL), "date_histogram.interval");
-                        String thisTimezone  = (String)agg.get(DateHistoGroupConfig.TIME_ZONE.getPreferredName());
+                        String thisTimezone = (String) agg.get(DateHistogramGroupConfig.TIME_ZONE);
                         String sourceTimeZone = source.timeZone() == null ? DateTimeZone.UTC.toString() : source.timeZone().toString();
 
                         // Ensure we are working on the same timezone
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java
index 538babf4fbced..0668e7c43ad3c 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.xpack.core.rollup.RollupField;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
 import org.joda.time.DateTimeZone;
 
 import java.util.ArrayList;
@@ -221,7 +221,7 @@ private static List<AggregationBuilder> translateDateHistogram(DateHistogramAggr
 
             String timezone = source.timeZone() == null ? DateTimeZone.UTC.toString() : source.timeZone().toString();
             filterConditions.add(new TermQueryBuilder(RollupField.formatFieldName(source,
-                    DateHistoGroupConfig.TIME_ZONE.getPreferredName()), timezone));
+                DateHistogramGroupConfig.TIME_ZONE), timezone));
 
             rolledDateHisto.offset(source.offset());
             if (source.extendedBounds() != null) {
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupIndexCaps.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupIndexCaps.java
index 22bf9ff06c26a..88c2986574792 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupIndexCaps.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupIndexCaps.java
@@ -75,7 +75,7 @@ public class RollupIndexCaps implements Writeable, ToXContentFragment {
 
                         // "job-1"
                         while (parser.nextToken().equals(XContentParser.Token.END_OBJECT) == false) {
-                            jobs.add(RollupJobConfig.PARSER.apply(parser, aVoid).build());
+                            jobs.add(RollupJobConfig.fromXContent(parser, null));
                         }
                     }
                 }
@@ -167,4 +167,4 @@ public boolean equals(Object other) {
     public int hashCode() {
         return Objects.hash(rollupIndexName, jobCaps);
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
index 7be9cc7ae3b20..c63ab96fa2595 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
@@ -56,7 +56,7 @@
 import org.elasticsearch.xpack.core.rollup.RollupField;
 import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps;
 import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
 import org.elasticsearch.xpack.rollup.Rollup;
 import org.elasticsearch.xpack.rollup.RollupJobIdentifierUtils;
 import org.elasticsearch.xpack.rollup.RollupRequestTranslator;
@@ -191,7 +191,9 @@ static MultiSearchRequest createMSearchRequest(SearchRequest request, NamedWrite
             copiedSource.query(new BoolQueryBuilder()
                     .must(rewritten)
                     .filter(new TermQueryBuilder(RollupField.formatMetaField(RollupField.ID.getPreferredName()), id))
-                    .filter(new TermQueryBuilder(RollupField.formatMetaField(RollupField.VERSION_FIELD), Rollup.ROLLUP_VERSION)));
+                    // Both versions are acceptable right now since they are compatible at search time
+                    .filter(new TermsQueryBuilder(RollupField.formatMetaField(RollupField.VERSION_FIELD),
+                        new long[]{Rollup.ROLLUP_VERSION_V1, Rollup.ROLLUP_VERSION_V2})));
 
             // And add a new msearch per JobID
             msearch.add(new SearchRequest(context.getRollupIndices(), copiedSource).types(request.types()));
@@ -328,10 +330,10 @@ private static String rewriteFieldName(Set<RollupJobCaps> jobCaps,
 
                         // If the cap is for a date_histo, and the query is a range, the timezones need to match
                         if (type.equals(DateHistogramAggregationBuilder.NAME) && timeZone != null) {
-                            boolean matchingTZ = ((String)agg.get(DateHistoGroupConfig.TIME_ZONE.getPreferredName()))
+                            boolean matchingTZ = ((String)agg.get(DateHistogramGroupConfig.TIME_ZONE))
                                 .equalsIgnoreCase(timeZone);
                             if (matchingTZ == false) {
-                                incompatibleTimeZones.add((String)agg.get(DateHistoGroupConfig.TIME_ZONE.getPreferredName()));
+                                incompatibleTimeZones.add((String)agg.get(DateHistogramGroupConfig.TIME_ZONE));
                             }
                             return matchingTZ;
                         }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java
index efac4c2d61b98..9119a5445d42e 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java
@@ -8,7 +8,6 @@
 import org.apache.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.common.Numbers;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
@@ -16,19 +15,17 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
 import org.elasticsearch.xpack.core.rollup.RollupField;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobStats;
 import org.elasticsearch.xpack.rollup.Rollup;
 
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
-import java.util.zip.CRC32;
 
 /**
  * These utilities are used to convert agg responses into a set of rollup documents.
@@ -41,12 +38,16 @@ class IndexerUtils {
      * The only entry point in this class.  You hand this method an aggregation and an index
      * pattern, and it returns a list of rolled documents that you can index
      *
-     * @param agg          The aggregation response that you want to rollup
-     * @param rollupIndex  The index that holds rollups for this job
+     * @param agg              The aggregation response that you want to rollup
+     * @param rollupIndex      The index that holds rollups for this job
+     * @param stats            The stats accumulator for this job's task
+     * @param groupConfig      The grouping configuration for the job
+     * @param jobId            The ID for the job
+     * @param isUpgradedDocID  `true` if this job is using the new ID scheme
      * @return             A list of rolled documents derived from the response
      */
     static List<IndexRequest> processBuckets(CompositeAggregation agg, String rollupIndex, RollupJobStats stats,
-                                             GroupConfig groupConfig, String jobId) {
+                                             GroupConfig groupConfig, String jobId, boolean isUpgradedDocID) {
 
         logger.debug("Buckets: [" + agg.getBuckets().size() + "][" + jobId + "]");
         return agg.getBuckets().stream().map(b ->{
@@ -57,24 +58,30 @@ static List<IndexRequest> processBuckets(CompositeAggregation agg, String rollup
             TreeMap<String, Object> keys = new TreeMap<>(b.getKey());
             List<Aggregation> metrics = b.getAggregations().asList();
 
+            RollupIDGenerator idGenerator;
+            if (isUpgradedDocID) {
+                idGenerator = new RollupIDGenerator.Murmur3(jobId);
+            } else  {
+                idGenerator = new RollupIDGenerator.CRC();
+            }
             Map<String, Object> doc = new HashMap<>(keys.size() + metrics.size());
-            CRC32 docId = processKeys(keys, doc, b.getDocCount(), groupConfig);
-            byte[] vs = jobId.getBytes(StandardCharsets.UTF_8);
-            docId.update(vs, 0, vs.length);
+
+            processKeys(keys, doc, b.getDocCount(), groupConfig, idGenerator);
+            idGenerator.add(jobId);
             processMetrics(metrics, doc);
 
-            doc.put(RollupField.ROLLUP_META + "." + RollupField.VERSION_FIELD, Rollup.ROLLUP_VERSION);
+            doc.put(RollupField.ROLLUP_META + "." + RollupField.VERSION_FIELD,
+                isUpgradedDocID ? Rollup.CURRENT_ROLLUP_VERSION : Rollup.ROLLUP_VERSION_V1);
             doc.put(RollupField.ROLLUP_META + "." + RollupField.ID.getPreferredName(), jobId);
 
-            IndexRequest request = new IndexRequest(rollupIndex, RollupField.TYPE_NAME, String.valueOf(docId.getValue()));
+            IndexRequest request = new IndexRequest(rollupIndex, RollupField.TYPE_NAME, idGenerator.getID());
             request.source(doc);
             return request;
         }).collect(Collectors.toList());
     }
 
-    private static CRC32 processKeys(Map<String, Object> keys, Map<String, Object> doc, long count, GroupConfig groupConfig) {
-        CRC32 docID = new CRC32();
-
+    private static void processKeys(Map<String, Object> keys, Map<String, Object> doc,
+                                     long count, GroupConfig groupConfig, RollupIDGenerator idGenerator) {
         keys.forEach((k, v) -> {
             // Also add a doc count for each key.  This will duplicate data, but makes search easier later
             doc.put(k + "." + RollupField.COUNT_FIELD, count);
@@ -82,38 +89,35 @@ private static CRC32 processKeys(Map<String, Object> keys, Map<String, Object> d
             if (k.endsWith("." + DateHistogramAggregationBuilder.NAME)) {
                 assert v != null;
                 doc.put(k + "." + RollupField.TIMESTAMP, v);
-                doc.put(k  + "." + RollupField.INTERVAL, groupConfig.getDateHisto().getInterval());
-                doc.put(k  + "." + DateHistoGroupConfig.TIME_ZONE, groupConfig.getDateHisto().getTimeZone().toString());
-                docID.update(Numbers.longToBytes((Long)v), 0, 8);
+                doc.put(k  + "." + RollupField.INTERVAL, groupConfig.getDateHistogram().getInterval());
+                doc.put(k  + "." + DateHistogramGroupConfig.TIME_ZONE, groupConfig.getDateHistogram().getTimeZone());
+                idGenerator.add((Long)v);
             } else if (k.endsWith("." + HistogramAggregationBuilder.NAME)) {
                 doc.put(k + "." + RollupField.VALUE, v);
-                doc.put(k + "." + RollupField.INTERVAL, groupConfig.getHisto().getInterval());
+                doc.put(k + "." + RollupField.INTERVAL, groupConfig.getHistogram().getInterval());
                 if (v == null) {
-                    // Arbitrary value to update the doc ID with for nulls
-                    docID.update(19);
+                    idGenerator.addNull();
                 } else {
-                    docID.update(Numbers.doubleToBytes((Double) v), 0, 8);
+                    idGenerator.add((Double) v);
                 }
             } else if (k.endsWith("." + TermsAggregationBuilder.NAME)) {
                 doc.put(k + "." + RollupField.VALUE, v);
                 if (v == null) {
-                    // Arbitrary value to update the doc ID with for nulls
-                    docID.update(19);
+                    idGenerator.addNull();
                 } else if (v instanceof String) {
-                    byte[] vs = ((String) v).getBytes(StandardCharsets.UTF_8);
-                    docID.update(vs, 0, vs.length);
+                    idGenerator.add((String)v);
                 } else if (v instanceof Long) {
-                    docID.update(Numbers.longToBytes((Long)v), 0, 8);
+                    idGenerator.add((Long)v);
                 } else if (v instanceof Double) {
-                    docID.update(Numbers.doubleToBytes((Double)v), 0, 8);
+                    idGenerator.add((Double)v);
                 } else {
-                    throw new RuntimeException("Encountered value of type [" + v.getClass() + "], which was unable to be processed.");
+                    throw new RuntimeException("Encountered value of type ["
+                        + v.getClass() + "], which was unable to be processed.");
                 }
             } else {
                 throw new ElasticsearchException("Could not identify key in agg [" + k + "]");
             }
         });
-        return docID;
     }
 
     private static void processMetrics(List<Aggregation> metrics, Map<String, Object> doc) {
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIDGenerator.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIDGenerator.java
new file mode 100644
index 0000000000000..3f9fc73873914
--- /dev/null
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIDGenerator.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.rollup.job;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.common.Numbers;
+import org.elasticsearch.common.hash.MurmurHash3;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.zip.CRC32;
+
+/**
+ * The ID Generator creates a deterministic document ID to be used for rollup docs.
+ * It does this by accepting values (e.g. composite keys for the rollup bucket) and
+ * hashes those together in a deterministic manner.
+ *
+ * Once the ID has generated, the generator instance becomes locked and will not
+ * accept any more values.  This is a safety mechanism to prevent accidentally
+ * changing the ID at runtime.
+ *
+ * NOTE: this class is not thread safe; there is no synchronization on the "generated"
+ * flag and it is unsafe to use from multiple threads.
+ */
+public abstract class RollupIDGenerator {
+    public abstract void add(Integer v);
+    public abstract void add(Long v);
+    public abstract void add(Double v);
+    public abstract void add(String v);
+    public abstract void addNull();
+    public abstract String getID();
+
+    private boolean generated = false;
+
+    final boolean check(Object v) {
+        if (generated) {
+            throw new RuntimeException("Cannot update ID as it has already been generated.");
+        }
+        if (v == null) {
+            addNull();
+            return false;
+        }
+        return true;
+    }
+
+    final void setFlag() {
+        if (generated) {
+            throw new RuntimeException("Cannot generate ID as it has already been generated.");
+        }
+        generated = true;
+    }
+
+    /**
+     * The "old" style ID used in Rollup V1.  A rolling 32 bit CRC.
+     *
+     * Null values are hashed as (int)19.
+     */
+    @Deprecated
+    public static class CRC extends RollupIDGenerator {
+        private final CRC32 crc = new CRC32();
+
+        @Override
+        public void add(Integer v) {
+            if (check(v)) {
+                crc.update(v);
+            }
+        }
+
+        @Override
+        public void add(Long v) {
+            if (check(v)) {
+                crc.update(Numbers.longToBytes(v), 0, 8);
+            }
+        }
+
+        @Override
+        public void add(Double v) {
+            if (check(v)) {
+                crc.update(Numbers.doubleToBytes(v), 0, 8);
+            }
+        }
+
+        @Override
+        public void add(String v) {
+            if (check(v)) {
+                byte[] vs = (v).getBytes(StandardCharsets.UTF_8);
+                crc.update(vs, 0, vs.length);
+            }
+        }
+
+        @Override
+        public void addNull() {
+            // Old ID scheme used (int)19 as the null placeholder.
+            // Not great but we're stuck with it :(
+            crc.update(19);
+        }
+
+        @Override
+        public String getID() {
+            setFlag();
+            return String.valueOf(crc.getValue());
+        }
+    }
+
+    /**
+     * The "new" style ID, used in Rollup V2.  A 128 bit Murmur3 hash of
+     * all the keys concatenated together, base64-encoded, then prepended
+     * with the job ID and a `$` delimiter
+     *
+     * Null values are hashed as a (hopefully) unique string `__NULL_PLACEHOLDER__830f1de2__`
+     */
+    public static class Murmur3 extends RollupIDGenerator {
+        private static final long SEED = 19;
+        private static final BytesRef DELIM = new BytesRef("$");
+        private static final BytesRef NULL_PLACEHOLDER = new BytesRef("__NULL_PLACEHOLDER__830f1de2__");
+        private final BytesRefBuilder id = new BytesRefBuilder();
+        private final String jobId;
+
+        Murmur3(String jobId) {
+            this.jobId = jobId;
+        }
+
+        @Override
+        public void add(Integer v) {
+            if (check(v)) {
+                update(Numbers.intToBytes(v));
+            }
+        }
+
+        @Override
+        public void add(Long v) {
+            if (check(v)) {
+                update(Numbers.longToBytes(v));
+            }
+        }
+
+        @Override
+        public void add(Double v) {
+            if (check(v)) {
+                update(Numbers.doubleToBytes(v));
+            }
+        }
+
+        @Override
+        public void add(String v) {
+            if (check(v)) {
+                update((v).getBytes(StandardCharsets.UTF_8));
+            }
+        }
+
+        @Override
+        public void addNull() {
+            // New ID scheme uses a (hopefully) unique placeholder for null
+            update(NULL_PLACEHOLDER.bytes);
+        }
+
+        private void update(byte[] v) {
+            id.append(v, 0, v.length);
+            id.append(DELIM);
+        }
+
+        @Override
+        public String getID() {
+            setFlag();
+            MurmurHash3.Hash128 hasher
+                = MurmurHash3.hash128(id.bytes(), 0, id.length(), SEED, new MurmurHash3.Hash128());
+            byte[] hashedBytes = new byte[16];
+            System.arraycopy(Numbers.longToBytes(hasher.h1), 0, hashedBytes, 0, 8);
+            System.arraycopy(Numbers.longToBytes(hasher.h2), 0, hashedBytes, 8, 8);
+            return jobId + "$" + Base64.getUrlEncoder().withoutPadding().encodeToString(hashedBytes);
+
+        }
+    }
+}
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
index c7d29451ab3be..87294706b3b7d 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.xpack.core.rollup.RollupField;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.IndexerState;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
@@ -34,6 +34,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
@@ -54,6 +55,7 @@ public abstract class RollupIndexer {
     private final AtomicReference<IndexerState> state;
     private final AtomicReference<Map<String, Object>> position;
     private final Executor executor;
+    protected final AtomicBoolean upgradedDocumentID;
 
     private final CompositeAggregationBuilder compositeBuilder;
     private long maxBoundary;
@@ -65,13 +67,15 @@ public abstract class RollupIndexer {
      * @param initialState Initial state for the indexer
      * @param initialPosition The last indexed bucket of the task
      */
-    RollupIndexer(Executor executor, RollupJob job, AtomicReference<IndexerState> initialState, Map<String, Object> initialPosition) {
+    RollupIndexer(Executor executor, RollupJob job, AtomicReference<IndexerState> initialState,
+                  Map<String, Object> initialPosition, AtomicBoolean upgradedDocumentID) {
         this.executor = executor;
         this.job = job;
         this.stats = new RollupJobStats();
         this.state = initialState;
         this.position = new AtomicReference<>(initialPosition);
         this.compositeBuilder = createCompositeBuilder(job.getConfig());
+        this.upgradedDocumentID = upgradedDocumentID;
     }
 
     /**
@@ -140,6 +144,13 @@ public RollupJobStats getStats() {
         return stats;
     }
 
+    /**
+     * Returns if this job has upgraded it's ID scheme yet or not
+     */
+    public boolean isUpgradedDocumentID() {
+        return upgradedDocumentID.get();
+    }
+
     /**
      * Sets the internal state to {@link IndexerState#STARTED} if the previous state was {@link IndexerState#STOPPED}. Setting the state to
      * STARTED allows a job to run in the background when {@link #maybeTriggerAsyncJob(long)} is called.
@@ -208,7 +219,7 @@ public synchronized boolean maybeTriggerAsyncJob(long now) {
 
                 // rounds the current time to its current bucket based on the date histogram interval.
                 // this is needed to exclude buckets that can still receive new documents.
-                DateHistoGroupConfig dateHisto = job.getConfig().getGroupConfig().getDateHisto();
+                DateHistogramGroupConfig dateHisto = job.getConfig().getGroupConfig().getDateHistogram();
                 long rounded = dateHisto.createRounding().round(now);
                 if (dateHisto.getDelay() != null) {
                     // if the job has a delay we filter all documents that appear before it.
@@ -312,8 +323,10 @@ private void onSearchResponse(SearchResponse searchResponse) {
             }
 
             final BulkRequest bulkRequest = new BulkRequest();
+            // Indexer is single-threaded, and only place that the ID scheme can get upgraded is doSaveState(), so
+            // we can pass down the boolean value rather than the atomic here
             final List<IndexRequest> docs = IndexerUtils.processBuckets(response, job.getConfig().getRollupIndex(),
-                    stats, job.getConfig().getGroupConfig(), job.getConfig().getId());
+                    stats, job.getConfig().getGroupConfig(), job.getConfig().getId(), upgradedDocumentID.get());
             docs.forEach(bulkRequest::add);
             assert bulkRequest.requests().size() > 0;
             doNextBulk(bulkRequest,
@@ -383,15 +396,14 @@ private CompositeAggregationBuilder createCompositeBuilder(RollupJobConfig confi
 
         // Add all the agg builders to our request in order: date_histo -> histo -> terms
         if (groupConfig != null) {
-            builders.addAll(groupConfig.getDateHisto().toBuilders());
-            metadata.putAll(groupConfig.getDateHisto().getMetadata());
-            if (groupConfig.getHisto() != null) {
-                builders.addAll(groupConfig.getHisto().toBuilders());
-                metadata.putAll(groupConfig.getHisto().getMetadata());
+            builders.addAll(groupConfig.getDateHistogram().toBuilders());
+            metadata.putAll(groupConfig.getDateHistogram().getMetadata());
+            if (groupConfig.getHistogram() != null) {
+                builders.addAll(groupConfig.getHistogram().toBuilders());
+                metadata.putAll(groupConfig.getHistogram().getMetadata());
             }
             if (groupConfig.getTerms() != null) {
                 builders.addAll(groupConfig.getTerms().toBuilders());
-                metadata.putAll(groupConfig.getTerms().getMetadata());
             }
         }
 
@@ -414,7 +426,7 @@ private CompositeAggregationBuilder createCompositeBuilder(RollupJobConfig confi
      */
     private QueryBuilder createBoundaryQuery(Map<String, Object> position) {
         assert maxBoundary < Long.MAX_VALUE;
-        DateHistoGroupConfig dateHisto = job.getConfig().getGroupConfig().getDateHisto();
+        DateHistogramGroupConfig dateHisto = job.getConfig().getGroupConfig().getDateHistogram();
         String fieldName = dateHisto.getField();
         String rollupFieldName = fieldName + "."  + DateHistogramAggregationBuilder.NAME;
         long lowerBound = 0L;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
index 50b3f21800d06..65362f9ad9dd3 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.xpack.rollup.Rollup;
 
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
@@ -97,8 +98,10 @@ protected class ClientRollupPageManager extends RollupIndexer {
         private final Client client;
         private final RollupJob job;
 
-        ClientRollupPageManager(RollupJob job, IndexerState initialState, Map<String, Object> initialPosition, Client client) {
-            super(threadPool.executor(ThreadPool.Names.GENERIC), job, new AtomicReference<>(initialState), initialPosition);
+        ClientRollupPageManager(RollupJob job, IndexerState initialState, Map<String, Object> initialPosition,
+                                Client client, AtomicBoolean upgradedDocumentID) {
+            super(threadPool.executor(ThreadPool.Names.GENERIC), job, new AtomicReference<>(initialState),
+                initialPosition, upgradedDocumentID);
             this.client = client;
             this.job = job;
         }
@@ -122,9 +125,16 @@ protected void doSaveState(IndexerState indexerState, Map<String, Object> positi
                 next.run();
             } else {
                 // Otherwise, attempt to persist our state
-                final RollupJobStatus state = new RollupJobStatus(indexerState, getPosition());
+                // Upgrade to the new ID scheme while we are at it
+                boolean oldState = upgradedDocumentID.getAndSet(true);
+                final RollupJobStatus state = new RollupJobStatus(indexerState, getPosition(), upgradedDocumentID.get());
                 logger.debug("Updating persistent state of job [" + job.getConfig().getId() + "] to [" + indexerState.toString() + "]");
-                updatePersistentTaskState(state, ActionListener.wrap(task -> next.run(), exc -> next.run()));
+                updatePersistentTaskState(state, ActionListener.wrap(task -> next.run(), exc -> {
+                    // We failed to update the persistent task for some reason,
+                    // set our flag back to what it was before
+                    upgradedDocumentID.set(oldState);
+                    next.run();
+                }));
             }
         }
 
@@ -148,6 +158,7 @@ protected void onAbort() {
     private final SchedulerEngine schedulerEngine;
     private final ThreadPool threadPool;
     private final RollupIndexer indexer;
+    private AtomicBoolean upgradedDocumentID;
 
     RollupJobTask(long id, String type, String action, TaskId parentTask, RollupJob job, RollupJobStatus state,
                   Client client, SchedulerEngine schedulerEngine, ThreadPool threadPool, Map<String, String> headers) {
@@ -156,6 +167,9 @@ protected void onAbort() {
         this.schedulerEngine = schedulerEngine;
         this.threadPool = threadPool;
 
+        // We can assume the new ID scheme only for new jobs
+        this.upgradedDocumentID = new AtomicBoolean(true);
+
         // If status is not null, we are resuming rather than starting fresh.
         Map<String, Object> initialPosition = null;
         IndexerState initialState = IndexerState.STOPPED;
@@ -169,25 +183,35 @@ protected void onAbort() {
                  * it is indexing but without the actual indexing thread running.
                  */
                 initialState = IndexerState.STARTED;
+
             } else if (existingState.equals(IndexerState.ABORTING) || existingState.equals(IndexerState.STOPPING)) {
                 // It shouldn't be possible to persist ABORTING, but if for some reason it does,
                 // play it safe and restore the job as STOPPED.  An admin will have to clean it up,
                 // but it won't be running, and won't delete itself either.  Safest option.
                 // If we were STOPPING, that means it persisted but was killed before finally stopped... so ok
-                // to restore as STOPEPD
+                // to restore as STOPPED
                 initialState = IndexerState.STOPPED;
             } else  {
                 initialState = existingState;
             }
             initialPosition = state.getPosition();
+
+            // Since we have state, we are resuming a job/checkpoint.  Although we are resuming
+            // from something that was checkpointed, we can't guarantee it was the _final_ checkpoint
+            // before the job ended (e.g. it could have been STOPPING, still indexing and killed, leaving
+            // us with an interval of time partially indexed).
+            //
+            // To be safe, if we are resuming any job, use it's ID upgrade status.  It will only
+            // be true if it actually finished a full checkpoint.
+            this.upgradedDocumentID.set(state.isUpgradedDocumentID());
         }
         this.indexer = new ClientRollupPageManager(job, initialState, initialPosition,
-                new ParentTaskAssigningClient(client, new TaskId(getPersistentTaskId())));
+                new ParentTaskAssigningClient(client, new TaskId(getPersistentTaskId())), upgradedDocumentID);
     }
 
     @Override
     public Status getStatus() {
-        return new RollupJobStatus(indexer.getState(), indexer.getPosition());
+        return new RollupJobStatus(indexer.getState(), indexer.getPosition(), upgradedDocumentID.get());
     }
 
     /**
@@ -223,13 +247,16 @@ public synchronized void start(ActionListener<StartRollupJobAction.Response> lis
                     + " state was [" + prevState + "]"));
             return;
         }
+
         final IndexerState newState = indexer.start();
         if (newState != IndexerState.STARTED) {
             listener.onFailure(new ElasticsearchException("Cannot start task for Rollup Job [" + job.getConfig().getId() + "] because"
                     + " state was [" + newState + "]"));
             return;
         }
-        final RollupJobStatus state = new RollupJobStatus(IndexerState.STARTED, indexer.getPosition());
+
+
+        final RollupJobStatus state = new RollupJobStatus(IndexerState.STARTED, indexer.getPosition(), upgradedDocumentID.get());
         logger.debug("Updating state for rollup job [" + job.getConfig().getId() + "] to [" + state.getIndexerState() + "][" +
                 state.getPosition() + "]");
         updatePersistentTaskState(state,
@@ -240,6 +267,8 @@ public synchronized void start(ActionListener<StartRollupJobAction.Response> lis
                             listener.onResponse(new StartRollupJobAction.Response(true));
                         },
                         (exc) -> {
+                            // We were unable to update the persistent status, so we need to shutdown the indexer too.
+                            indexer.stop();
                             listener.onFailure(
                                     new ElasticsearchException("Error while updating state for rollup job [" + job.getConfig().getId()
                                             + "] to [" + state.getIndexerState() + "].", exc)
@@ -261,6 +290,7 @@ public synchronized void start(ActionListener<StartRollupJobAction.Response> lis
      * @param listener The listener that is requesting the stop, so that we can signal completion/failure
      */
     public synchronized void stop(ActionListener<StopRollupJobAction.Response> listener) {
+
         final IndexerState newState = indexer.stop();
         switch (newState) {
             case STOPPED:
@@ -268,9 +298,13 @@ public synchronized void stop(ActionListener<StopRollupJobAction.Response> liste
                 break;
 
             case STOPPING:
-                // update the persistent state only if there is no background job running,
-                // otherwise the state is updated by the indexer when the background job detects the STOPPING state.
-                RollupJobStatus state = new RollupJobStatus(IndexerState.STOPPED, indexer.getPosition());
+                // update the persistent state to STOPPED.  There are two scenarios and both are safe:
+                // 1. we persist STOPPED now, indexer continues a bit then sees the flag and checkpoints another
+                //    STOPPED with the more recent position.  That will also upgrade the ID scheme
+                // 2. we persist STOPPED now, indexer continues a bit but then dies.  When/if we resume we'll pick up
+                //    at last checkpoint, overwrite some docs and eventually checkpoint.  At that time we'll also
+                //    upgrade the ID scheme
+                RollupJobStatus state = new RollupJobStatus(IndexerState.STOPPED, indexer.getPosition(), upgradedDocumentID.get());
                 updatePersistentTaskState(state,
                         ActionListener.wrap(
                                 (task) -> {
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java
index f41074e8e658f..231e382827e8a 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java
@@ -32,7 +32,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
         String id = restRequest.param(ID.getPreferredName());
         XContentParser parser = restRequest.contentParser();
 
-        PutRollupJobAction.Request request = PutRollupJobAction.Request.parseRequest(id, parser);
+        PutRollupJobAction.Request request = PutRollupJobAction.Request.fromXContent(parser, id);
 
         return channel -> client.execute(PutRollupJobAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java
index d88042188a0fb..3235d0c39e256 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java
@@ -15,45 +15,43 @@
 import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
-import org.elasticsearch.xpack.core.rollup.job.HistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.HistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
+import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
 import org.joda.time.DateTimeZone;
 
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.equalTo;
 
 public class RollupJobIdentifierUtilTests extends ESTestCase {
 
     public void testOneMatch() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
         Set<RollupJobCaps> caps = singletonSet(cap);
 
         DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo")
-                .dateHistogramInterval(job.getGroupConfig().getDateHisto().getInterval());
+                .dateHistogramInterval(job.getGroupConfig().getDateHistogram().getInterval());
 
         Set<RollupJobCaps> bestCaps = RollupJobIdentifierUtils.findBestJobs(builder, caps);
         assertThat(bestCaps.size(), equalTo(1));
     }
 
     public void testBiggerButCompatibleInterval() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
         Set<RollupJobCaps> caps = singletonSet(cap);
 
         DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo")
@@ -64,11 +62,9 @@ public void testBiggerButCompatibleInterval() {
     }
 
     public void testIncompatibleInterval() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1d")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
         Set<RollupJobCaps> caps = singletonSet(cap);
 
         DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo")
@@ -80,12 +76,9 @@ public void testIncompatibleInterval() {
     }
 
     public void testBadTimeZone() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1d"))
-                .setTimeZone(DateTimeZone.forID("EST")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"), null, "EST"));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
         Set<RollupJobCaps> caps = singletonSet(cap);
 
         DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo")
@@ -98,15 +91,10 @@ public void testBadTimeZone() {
     }
 
     public void testMetricOnlyAgg() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job.setGroupConfig(group.build());
-        job.setMetricsConfig(Collections.singletonList(new MetricConfig.Builder()
-                .setField("bar")
-                .setMetrics(Collections.singletonList("max"))
-            .build()));
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final List<MetricConfig> metrics = singletonList(new MetricConfig("bar", singletonList("max")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, metrics, null);
+        RollupJobCaps cap = new RollupJobCaps(job);
         Set<RollupJobCaps> caps = singletonSet(cap);
 
         MaxAggregationBuilder max = new MaxAggregationBuilder("the_max").field("bar");
@@ -116,11 +104,9 @@ public void testMetricOnlyAgg() {
     }
 
     public void testOneOfTwoMatchingCaps() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
         Set<RollupJobCaps> caps = singletonSet(cap);
 
         DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo")
@@ -133,24 +119,16 @@ public void testOneOfTwoMatchingCaps() {
     }
 
     public void testTwoJobsSameRollupIndex() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        group.setTerms(null);
-        group.setHisto(null);
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
         Set<RollupJobCaps> caps = new HashSet<>(2);
         caps.add(cap);
 
-        RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2");
-        GroupConfig.Builder group2 = ConfigTestHelpers.getGroupConfig();
-        group2.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        group2.setTerms(null);
-        group2.setHisto(null);
-        job2.setGroupConfig(group.build());
-        job2.setRollupIndex(job.getRollupIndex());
-        RollupJobCaps cap2 = new RollupJobCaps(job2.build());
+        final GroupConfig group2 = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig job2 =
+            new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10,  group2, emptyList(), null);
+        RollupJobCaps cap2 = new RollupJobCaps(job2);
         caps.add(cap2);
 
         DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo")
@@ -163,27 +141,16 @@ public void testTwoJobsSameRollupIndex() {
     }
 
     public void testTwoJobsButBothPartialMatches() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job.setGroupConfig(group.build());
-        job.setMetricsConfig(Collections.singletonList(new MetricConfig.Builder()
-                .setField("bar")
-                .setMetrics(Collections.singletonList("max"))
-                .build()));
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final List<MetricConfig> metrics = singletonList(new MetricConfig("bar", singletonList("max")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, metrics, null);
+        RollupJobCaps cap = new RollupJobCaps(job);
         Set<RollupJobCaps> caps = new HashSet<>(2);
         caps.add(cap);
 
-        RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2");
-        GroupConfig.Builder group2 = ConfigTestHelpers.getGroupConfig();
-        group2.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job2.setGroupConfig(group.build());
-        job.setMetricsConfig(Collections.singletonList(new MetricConfig.Builder()
-                .setField("bar")
-                .setMetrics(Collections.singletonList("min"))
-                .build()));
-        RollupJobCaps cap2 = new RollupJobCaps(job2.build());
+        // TODO Is it what we really want to test?
+        final RollupJobConfig job2 = new RollupJobConfig("foo2", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        RollupJobCaps cap2 = new RollupJobCaps(job2);
         caps.add(cap2);
 
         DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo")
@@ -197,21 +164,14 @@ public void testTwoJobsButBothPartialMatches() {
     }
 
     public void testComparableDifferentDateIntervals() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build())
-                .setHisto(null)
-                .setTerms(null);
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
-
-        RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex());
-        GroupConfig.Builder group2 = ConfigTestHelpers.getGroupConfig();
-        group2.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1d")).build())
-                .setHisto(null)
-                .setTerms(null);
-        job2.setGroupConfig(group2.build());
-        RollupJobCaps cap2 = new RollupJobCaps(job2.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
+
+        final GroupConfig group2 = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d")));
+        final RollupJobConfig job2 =
+            new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10,  group2, emptyList(), null);
+        RollupJobCaps cap2 = new RollupJobCaps(job2);
 
         DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo")
                 .dateHistogramInterval(new DateHistogramInterval("1d"));
@@ -226,21 +186,14 @@ public void testComparableDifferentDateIntervals() {
     }
 
     public void testComparableDifferentDateIntervalsOnlyOneWorks() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build())
-                .setHisto(null)
-                .setTerms(null);
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
-
-        RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex());
-        GroupConfig.Builder group2 = ConfigTestHelpers.getGroupConfig();
-        group2.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1d")).build())
-                .setHisto(null)
-                .setTerms(null);
-        job2.setGroupConfig(group2.build());
-        RollupJobCaps cap2 = new RollupJobCaps(job2.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
+
+        final GroupConfig group2 = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d")));
+        final RollupJobConfig job2 =
+            new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10,  group2, emptyList(), null);
+        RollupJobCaps cap2 = new RollupJobCaps(job2);
 
         DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo")
                 .dateHistogramInterval(new DateHistogramInterval("1h"));
@@ -255,21 +208,15 @@ public void testComparableDifferentDateIntervalsOnlyOneWorks() {
     }
 
     public void testComparableNoHistoVsHisto() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build())
-                .setHisto(null)
-                .setTerms(null);
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
-
-        RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex());
-        GroupConfig.Builder group2 = ConfigTestHelpers.getGroupConfig();
-        group2.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build())
-                .setHisto(ConfigTestHelpers.getHisto().setInterval(100).setFields(Collections.singletonList("bar")).build())
-                .setTerms(null);
-        job2.setGroupConfig(group2.build());
-        RollupJobCaps cap2 = new RollupJobCaps(job2.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
+
+        final HistogramGroupConfig histoConfig = new HistogramGroupConfig(100L, "bar");
+        final GroupConfig group2 = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")), histoConfig, null);
+        final RollupJobConfig job2 =
+            new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10,  group2, emptyList(), null);
+        RollupJobCaps cap2 = new RollupJobCaps(job2);
 
         DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo")
                 .dateHistogramInterval(new DateHistogramInterval("1h"))
@@ -285,21 +232,15 @@ public void testComparableNoHistoVsHisto() {
     }
 
     public void testComparableNoTermsVsTerms() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build())
-                .setHisto(null)
-                .setTerms(null);
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
-
-        RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex());
-        GroupConfig.Builder group2 = ConfigTestHelpers.getGroupConfig();
-        group2.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build())
-                .setHisto(null)
-                .setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("bar")).build());
-        job2.setGroupConfig(group2.build());
-        RollupJobCaps cap2 = new RollupJobCaps(job2.build());
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
+
+        final TermsGroupConfig termsConfig = new TermsGroupConfig("bar");
+        final GroupConfig group2 = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")), null, termsConfig);
+        final RollupJobConfig job2 =
+            new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10,  group2, emptyList(), null);
+        RollupJobCaps cap2 = new RollupJobCaps(job2);
 
         DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo")
                 .dateHistogramInterval(new DateHistogramInterval("1h"))
@@ -321,25 +262,16 @@ public void testHistoSameNameWrongTypeInCaps() {
                 .subAggregation(new MaxAggregationBuilder("the_max").field("max_field"))
                 .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field"));
 
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo")
-                .setGroupConfig(ConfigTestHelpers.getGroupConfig()
-                        .setDateHisto(new DateHistoGroupConfig.Builder()
-                                .setInterval(new DateHistogramInterval("1d"))
-                                .setField("foo") // <-- NOTE same name but wrong type
-                                .setTimeZone(DateTimeZone.UTC)
-                                .build())
-                        .setHisto(new HistoGroupConfig.Builder()
-                                .setFields(Collections.singletonList("baz")) // <-- NOTE right type but wrong name
-                                .setInterval(1L)
-                                .build())
-                        .build())
-                .setMetricsConfig(Arrays.asList(new MetricConfig.Builder()
-                                .setField("max_field")
-                                .setMetrics(Collections.singletonList("max")).build(),
-                        new MetricConfig.Builder()
-                                .setField("avg_field")
-                                .setMetrics(Collections.singletonList("avg")).build()))
-                .build();
+        final GroupConfig group = new GroupConfig(
+                    // NOTE same name but wrong type
+                    new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID()),
+                    new HistogramGroupConfig(1L, "baz"), // <-- NOTE right type but wrong name
+                    null
+                );
+        final List<MetricConfig> metrics =
+            Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")));
+
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, metrics, null);
         Set<RollupJobCaps> caps = singletonSet(new RollupJobCaps(job));
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RollupJobIdentifierUtils.findBestJobs(histo, caps));
@@ -354,21 +286,13 @@ public void testMissingDateHisto() {
                 .subAggregation(new MaxAggregationBuilder("the_max").field("max_field"))
                 .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field"));
 
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo")
-                .setGroupConfig(ConfigTestHelpers.getGroupConfig()
-                        .setDateHisto(new DateHistoGroupConfig.Builder()
-                                .setInterval(new DateHistogramInterval("1d"))
-                                .setField("foo")
-                                .setTimeZone(DateTimeZone.UTC)
-                                .build())
-                        .build())
-                .setMetricsConfig(Arrays.asList(new MetricConfig.Builder()
-                                .setField("max_field")
-                                .setMetrics(Collections.singletonList("max")).build(),
-                        new MetricConfig.Builder()
-                                .setField("avg_field")
-                                .setMetrics(Collections.singletonList("avg")).build()))
-                .build();
+        final GroupConfig group = new GroupConfig(
+                    new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID())
+                );
+        final List<MetricConfig> metrics =
+                    Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")));
+
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, metrics, null);
         Set<RollupJobCaps> caps = singletonSet(new RollupJobCaps(job));
 
         Exception e = expectThrows(IllegalArgumentException.class, () -> RollupJobIdentifierUtils.findBestJobs(histo,caps));
@@ -383,15 +307,11 @@ public void testNoMatchingInterval() {
                 .subAggregation(new MaxAggregationBuilder("the_max").field("max_field"))
                 .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field"));
 
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo")
-                .setGroupConfig(ConfigTestHelpers.getGroupConfig()
-                        .setDateHisto(new DateHistoGroupConfig.Builder()
-                                .setInterval(new DateHistogramInterval("100d")) // <- interval in job is much higher than agg interval above
-                                .setField("foo")
-                                .setTimeZone(DateTimeZone.UTC)
-                                .build())
-                        .build())
-                .build();
+        final GroupConfig group = new GroupConfig(
+                    // interval in job is much higher than agg interval above
+                    new DateHistogramGroupConfig("foo", new DateHistogramInterval("100d"), null, DateTimeZone.UTC.getID())
+                );
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
         Set<RollupJobCaps> caps = singletonSet(new RollupJobCaps(job));
 
         Exception e = expectThrows(RuntimeException.class, () -> RollupJobIdentifierUtils.findBestJobs(histo, caps));
@@ -406,21 +326,14 @@ public void testDateHistoMissingFieldInCaps() {
                 .subAggregation(new MaxAggregationBuilder("the_max").field("max_field"))
                 .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field"));
 
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo")
-                .setGroupConfig(ConfigTestHelpers.getGroupConfig()
-                        .setDateHisto(new DateHistoGroupConfig.Builder()
-                                .setInterval(new DateHistogramInterval("1d"))
-                                .setField("bar") // <-- NOTE different field from the one in the query
-                                .setTimeZone(DateTimeZone.UTC)
-                                .build())
-                        .build())
-                .setMetricsConfig(Arrays.asList(new MetricConfig.Builder()
-                                .setField("max_field")
-                                .setMetrics(Collections.singletonList("max")).build(),
-                        new MetricConfig.Builder()
-                                .setField("avg_field")
-                                .setMetrics(Collections.singletonList("avg")).build()))
-                .build();
+        final GroupConfig group = new GroupConfig(
+                    // NOTE different field from the one in the query
+                    new DateHistogramGroupConfig("bar", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID())
+                );
+        final List<MetricConfig> metrics =
+                    Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")));
+
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, metrics, null);
         Set<RollupJobCaps> caps = singletonSet(new RollupJobCaps(job));
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RollupJobIdentifierUtils.findBestJobs(histo, caps));
@@ -435,25 +348,15 @@ public void testHistoMissingFieldInCaps() {
                 .subAggregation(new MaxAggregationBuilder("the_max").field("max_field"))
                 .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field"));
 
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo")
-                .setGroupConfig(ConfigTestHelpers.getGroupConfig()
-                        .setDateHisto(new DateHistoGroupConfig.Builder()
-                                .setInterval(new DateHistogramInterval("1d"))
-                                .setField("bar")
-                                .setTimeZone(DateTimeZone.UTC)
-                                .build())
-                        .setHisto(new HistoGroupConfig.Builder()
-                                .setFields(Collections.singletonList("baz")) // <-- NOTE note different field from one used in query
-                                .setInterval(1L)
-                                .build())
-                        .build())
-                .setMetricsConfig(Arrays.asList(new MetricConfig.Builder()
-                                .setField("max_field")
-                                .setMetrics(Collections.singletonList("max")).build(),
-                        new MetricConfig.Builder()
-                                .setField("avg_field")
-                                .setMetrics(Collections.singletonList("avg")).build()))
-                .build();
+        final GroupConfig group = new GroupConfig(
+                    new DateHistogramGroupConfig("bar", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID()),
+                    new HistogramGroupConfig(1L, "baz"), // <-- NOTE right type but wrong name
+                    null
+                );
+        final List<MetricConfig> metrics =
+                    Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")));
+
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, metrics, null);
         Set<RollupJobCaps> caps = singletonSet(new RollupJobCaps(job));
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RollupJobIdentifierUtils.findBestJobs(histo, caps));
@@ -467,20 +370,13 @@ public void testNoMatchingHistoInterval() {
                 .field("bar")
                 .subAggregation(new MaxAggregationBuilder("the_max").field("max_field"))
                 .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field"));
-        
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo")
-                .setGroupConfig(ConfigTestHelpers.getGroupConfig()
-                        .setDateHisto(new DateHistoGroupConfig.Builder()
-                                .setInterval(new DateHistogramInterval("1d"))
-                                .setField("foo")
-                                .setTimeZone(DateTimeZone.UTC)
-                                .build())
-                        .setHisto(new HistoGroupConfig.Builder()
-                                .setFields(Collections.singletonList("bar"))
-                                .setInterval(100L) // <--- interval in job is much higher than agg interval above
-                                .build())
-                        .build())
-                .build();
+
+        final GroupConfig group = new GroupConfig(
+                    new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID()),
+                    new HistogramGroupConfig(1L, "baz"), // <-- NOTE right type but wrong name
+                    null
+                );
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
         Set<RollupJobCaps> caps = singletonSet(new RollupJobCaps(job));
 
         Exception e = expectThrows(RuntimeException.class,
@@ -492,11 +388,10 @@ public void testNoMatchingHistoInterval() {
     public void testMissingMetric() {
         int i = ESTestCase.randomIntBetween(0, 3);
 
-        Set<RollupJobCaps> caps = singletonSet(new RollupJobCaps(ConfigTestHelpers
-                .getRollupJob("foo").setMetricsConfig(Collections.singletonList(new MetricConfig.Builder()
-                        .setField("foo")
-                        .setMetrics(Arrays.asList("avg", "max", "min", "sum")).build()))
-                .build()));
+        final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final List<MetricConfig> metrics = singletonList(new MetricConfig("foo", Arrays.asList("avg", "max", "min", "sum")));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  group, emptyList(), null);
+        Set<RollupJobCaps> caps = singletonSet(new RollupJobCaps(job));
 
         String aggType;
         Exception e;
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java
index 9b29c907b3bed..a618e8b4e6f63 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java
@@ -27,20 +27,14 @@
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
-import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps;
-import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -152,12 +146,6 @@ public void testSimpleMetric() {
     }
 
     public void testUnsupportedMetric() {
-        Set<RollupJobCaps> caps = singletonSet(new RollupJobCaps(ConfigTestHelpers
-                .getRollupJob("foo").setMetricsConfig(Collections.singletonList(new MetricConfig.Builder()
-                        .setField("foo")
-                        .setMetrics(Arrays.asList("avg", "max", "min", "sum")).build()))
-                .build()));
-
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> translateAggregation(new StatsAggregationBuilder("test_metric")
                         .field("foo"), Collections.emptyList(), namedWriteableRegistry));
@@ -384,10 +372,4 @@ public void testUnsupportedAgg() {
         assertThat(e.getMessage(), equalTo("Unable to translate aggregation tree into Rollup.  Aggregation [test_geo] is of type " +
                 "[GeoDistanceAggregationBuilder] which is currently unsupported."));
     }
-    
-    private Set<RollupJobCaps> singletonSet(RollupJobCaps cap) {
-        Set<RollupJobCaps> caps = new HashSet<>();
-        caps.add(cap);
-        return caps;
-    }
 }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java
index 419feb6f19c0b..bf73ceb88113e 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java
@@ -70,7 +70,7 @@ public void testStateCheckNoMatchingPersistentTasks() {
 
     public void testStateCheckMatchingPersistentTasks() {
         GetRollupJobsAction.Request request = new GetRollupJobsAction.Request("foo");
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"), Collections.emptyMap());
         Map<String, PersistentTasksCustomMetaData.PersistentTask<?>> tasks
                 = Collections.singletonMap("foo", new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupJob.NAME, job, 1, null));
         ClusterState state = ClusterState.builder(new ClusterName("_name"))
@@ -83,7 +83,7 @@ public void testStateCheckMatchingPersistentTasks() {
 
     public void testStateCheckAllMatchingPersistentTasks() {
         GetRollupJobsAction.Request request = new GetRollupJobsAction.Request("_all");
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"), Collections.emptyMap());
         Map<String, PersistentTasksCustomMetaData.PersistentTask<?>> tasks
                 = Collections.singletonMap("foo", new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupJob.NAME, job, 1, null));
         ClusterState state = ClusterState.builder(new ClusterName("_name"))
@@ -96,8 +96,8 @@ public void testStateCheckAllMatchingPersistentTasks() {
 
     public void testStateCheckAllWithSeveralMatchingPersistentTasks() {
         GetRollupJobsAction.Request request = new GetRollupJobsAction.Request("_all");
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
-        RollupJob job2 = new RollupJob(ConfigTestHelpers.getRollupJob("bar").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"), Collections.emptyMap());
+        RollupJob job2 = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "bar"), Collections.emptyMap());
         Map<String, PersistentTasksCustomMetaData.PersistentTask<?>> tasks = new HashMap<>(2);
         tasks.put("foo", new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupJob.NAME, job, 1, null));
         tasks.put("bar", new PersistentTasksCustomMetaData.PersistentTask<>("bar", RollupJob.NAME, job2, 1, null));
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java
index e3a45dbd66b87..9068bcfce36a4 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java
@@ -89,7 +89,7 @@ public void testMissingJob() throws IOException {
     public void testOneJob() throws IOException {
         String indexPattern = randomBoolean() ? randomAlphaOfLength(10) : randomAlphaOfLength(10) + "-*";
         String jobName = randomAlphaOfLength(5);
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob(jobName).build();
+        RollupJobConfig job = ConfigTestHelpers.randomRollupJobConfig(random(), jobName);
 
         MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
                 Collections.singletonMap(RollupField.TYPE_NAME,
@@ -113,7 +113,7 @@ public void testMultipleJobs() throws IOException {
         Map<String, Object> jobs = new HashMap<>(num);
         for (int i = 0; i < num; i++) {
             String jobName = randomAlphaOfLength(5);
-            jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).build());
+            jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName));
         }
 
         MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
@@ -147,7 +147,7 @@ public void testAllIndices() throws IOException {
                 String jobName = randomAlphaOfLength(10);
                 String indexName = Integer.toString(indexCounter);
                 indexCounter += 1;
-                jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).setIndexPattern(indexName).build());
+                jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, indexName));
             }
 
             MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
@@ -179,7 +179,7 @@ public void testOneIndex() throws IOException {
             Map<String, Object> jobs = new HashMap<>(num);
             for (int i = 0; i < num; i++) {
                 String jobName = randomAlphaOfLength(5);
-                jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).setIndexPattern(indexName).build());
+                jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, indexName));
             }
 
             MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java
index 2066d6649965f..e9d5d6153b18c 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java
@@ -60,7 +60,7 @@ public void testAllIndicesByRollupSingleRollup() throws IOException {
                 String jobName = randomAlphaOfLength(10);
                 String indexName = Integer.toString(indexCounter);
                 indexCounter += 1;
-                jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).setRollupIndex("foo").build());
+                jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, indexName, "foo"));
             }
 
             MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
@@ -89,10 +89,7 @@ public void testAllIndicesByRollupManyRollup() throws IOException {
             String jobName = randomAlphaOfLength(10);
             String indexName = Integer.toString(indexCounter);
             indexCounter += 1;
-            jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName)
-                .setIndexPattern(indexName)
-                .setRollupIndex("rollup_" + indexName).build());
-
+            jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, indexName, "rollup_" + indexName));
 
             MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
                 Collections.singletonMap(RollupField.TYPE_NAME,
@@ -120,9 +117,7 @@ public void testOneIndexByRollupManyRollup() throws IOException {
             String jobName = randomAlphaOfLength(10);
             String indexName = Integer.toString(indexCounter);
             indexCounter += 1;
-            jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName)
-                .setIndexPattern("foo_" + indexName)
-                .setRollupIndex("rollup_" + indexName).build());
+            jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, "foo_" + indexName, "rollup_" + indexName));
 
             MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
                 Collections.singletonMap(RollupField.TYPE_NAME,
@@ -151,9 +146,7 @@ public void testOneIndexByRollupOneRollup() throws IOException {
             String jobName = randomAlphaOfLength(10);
             String indexName = Integer.toString(indexCounter);
             indexCounter += 1;
-            jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName)
-                .setIndexPattern("foo_" + indexName)
-                .setRollupIndex("rollup_foo").build());
+            jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, "foo_" + indexName, "rollup_foo"));
 
             MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
                 Collections.singletonMap(RollupField.TYPE_NAME,
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java
index 254c2a8c81194..848bd5f13dd9e 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java
@@ -12,6 +12,8 @@
 import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.junit.Before;
 
+import java.io.IOException;
+
 public class PutJobActionRequestTests extends AbstractStreamableXContentTestCase<Request> {
 
     private String jobId;
@@ -23,7 +25,7 @@ public void setupJobID() {
 
     @Override
     protected Request createTestInstance() {
-        return new Request(ConfigTestHelpers.getRollupJob(jobId).build());
+        return new Request(ConfigTestHelpers.randomRollupJobConfig(random(), jobId));
     }
 
     @Override
@@ -37,9 +39,8 @@ protected Request createBlankInstance() {
     }
 
     @Override
-    protected Request doParseInstance(XContentParser parser) {
-        return Request.parseRequest(jobId, parser);
+    protected Request doParseInstance(final XContentParser parser) throws IOException {
+        return Request.fromXContent(parser, jobId);
     }
 
-
 }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
index 58fa9d4533bc3..d9caad5147d41 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
@@ -50,7 +50,7 @@ public class PutJobStateMachineTests extends ESTestCase {
 
     @SuppressWarnings("unchecked")
     public void testCreateIndexException() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"), Collections.emptyMap());
 
         ActionListener<PutRollupJobAction.Response> testListener = ActionListener.wrap(response -> {
             fail("Listener success should not have been triggered.");
@@ -76,7 +76,7 @@ public void testCreateIndexException() {
 
     @SuppressWarnings("unchecked")
     public void testIndexAlreadyExists() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
 
         ActionListener<PutRollupJobAction.Response> testListener = ActionListener.wrap(response -> {
             fail("Listener success should not have been triggered.");
@@ -108,7 +108,7 @@ public void testIndexAlreadyExists() {
 
     @SuppressWarnings("unchecked")
     public void testIndexMetaData() throws InterruptedException {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
 
         ActionListener<PutRollupJobAction.Response> testListener = ActionListener.wrap(response -> {
             fail("Listener success should not have been triggered.");
@@ -151,7 +151,7 @@ public void testIndexMetaData() throws InterruptedException {
 
     @SuppressWarnings("unchecked")
     public void testGetMappingFails() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"), Collections.emptyMap());
 
         ActionListener<PutRollupJobAction.Response> testListener = ActionListener.wrap(response -> {
             fail("Listener success should not have been triggered.");
@@ -175,13 +175,13 @@ public void testGetMappingFails() {
 
     @SuppressWarnings("unchecked")
     public void testNoMetadataInMapping() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
 
         ActionListener<PutRollupJobAction.Response> testListener = ActionListener.wrap(response -> {
             fail("Listener success should not have been triggered.");
         }, e -> {
             assertThat(e.getMessage(), equalTo("Expected to find _meta key in mapping of rollup index ["
-                    + job.getConfig().getRollupIndex() + "] but not found."));
+                + job.getConfig().getRollupIndex() + "] but not found."));
         });
 
         Logger logger = mock(Logger.class);
@@ -208,7 +208,7 @@ public void testNoMetadataInMapping() {
 
     @SuppressWarnings("unchecked")
     public void testNoMappingVersion() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
 
         ActionListener<PutRollupJobAction.Response> testListener = ActionListener.wrap(response -> {
             fail("Listener success should not have been triggered.");
@@ -245,7 +245,7 @@ public void testNoMappingVersion() {
 
     @SuppressWarnings("unchecked")
     public void testJobAlreadyInMapping() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"), Collections.emptyMap());
 
         ActionListener<PutRollupJobAction.Response> testListener = ActionListener.wrap(response -> {
             fail("Listener success should not have been triggered.");
@@ -282,12 +282,12 @@ public void testJobAlreadyInMapping() {
 
     @SuppressWarnings("unchecked")
     public void testAddJobToMapping() {
-        RollupJobConfig unrelatedJob = ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLength(10))
-                .setIndexPattern("foo").setRollupIndex("rollup_index_foo").build();
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo")
-                .setIndexPattern("foo")
-                .setRollupIndex("rollup_index_foo")
-                .build(), Collections.emptyMap());
+        final RollupJobConfig unrelatedJob =
+            ConfigTestHelpers.randomRollupJobConfig(random(), ESTestCase.randomAlphaOfLength(10), "foo", "rollup_index_foo");
+
+        final RollupJobConfig config =
+            ConfigTestHelpers.randomRollupJobConfig(random(), ESTestCase.randomAlphaOfLength(10), "foo", "rollup_index_foo");
+        RollupJob job = new RollupJob(config, Collections.emptyMap());
         ActionListener<PutRollupJobAction.Response> testListener = ActionListener.wrap(response -> {
             fail("Listener success should not have been triggered.");
         }, e -> {
@@ -331,7 +331,7 @@ public void testAddJobToMapping() {
 
     @SuppressWarnings("unchecked")
     public void testTaskAlreadyExists() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"), Collections.emptyMap());
 
         ActionListener<PutRollupJobAction.Response> testListener = ActionListener.wrap(response -> {
             fail("Listener success should not have been triggered.");
@@ -346,7 +346,7 @@ public void testTaskAlreadyExists() {
             requestCaptor.getValue().onFailure(new ResourceAlreadyExistsException(job.getConfig().getRollupIndex()));
             return null;
         }).when(tasksService).sendStartRequest(eq(job.getConfig().getId()),
-                eq(RollupField.TASK_NAME), eq(job), requestCaptor.capture());
+            eq(RollupField.TASK_NAME), eq(job), requestCaptor.capture());
 
         TransportPutRollupJobAction.startPersistentTask(job, testListener, tasksService);
         verify(tasksService).sendStartRequest(eq(job.getConfig().getId()), eq(RollupField.TASK_NAME), eq(job), any());
@@ -354,7 +354,7 @@ public void testTaskAlreadyExists() {
 
     @SuppressWarnings("unchecked")
     public void testStartTask() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
 
         ActionListener<PutRollupJobAction.Response> testListener = ActionListener.wrap(response -> {
             fail("Listener success should not have been triggered.");
@@ -367,14 +367,14 @@ public void testStartTask() {
         ArgumentCaptor<ActionListener> requestCaptor = ArgumentCaptor.forClass(ActionListener.class);
         doAnswer(invocation -> {
             PersistentTasksCustomMetaData.PersistentTask<RollupJob> response
-                    = new PersistentTasksCustomMetaData.PersistentTask<>(job.getConfig().getId(), RollupField.TASK_NAME, job, 123,
-                    mock(PersistentTasksCustomMetaData.Assignment.class));
+                = new PersistentTasksCustomMetaData.PersistentTask<>(job.getConfig().getId(), RollupField.TASK_NAME, job, 123,
+                mock(PersistentTasksCustomMetaData.Assignment.class));
             requestCaptor.getValue().onResponse(response);
             return null;
         }).when(tasksService).sendStartRequest(eq(job.getConfig().getId()), eq(RollupField.TASK_NAME), eq(job), requestCaptor.capture());
 
         ArgumentCaptor<PersistentTasksService.WaitForPersistentTaskListener> requestCaptor2
-                = ArgumentCaptor.forClass(PersistentTasksService.WaitForPersistentTaskListener.class);
+            = ArgumentCaptor.forClass(PersistentTasksService.WaitForPersistentTaskListener.class);
         doAnswer(invocation -> {
             // Bail here with an error, further testing will happen through tests of #startPersistentTask
             requestCaptor2.getValue().onFailure(new RuntimeException("Ending"));
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/RollupIndexCapsTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/RollupIndexCapsTests.java
index a1e4dba0fffda..78b1e1e0d2d0c 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/RollupIndexCapsTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/RollupIndexCapsTests.java
@@ -30,8 +30,8 @@ public void testSetEmptyJobs() {
 
     public void testGetAllJobs() {
         List<RollupJobConfig> jobs = new ArrayList<>(2);
-        jobs.add(ConfigTestHelpers.getRollupJob("foo").build());
-        jobs.add(ConfigTestHelpers.getRollupJob("bar").build());
+        jobs.add(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"));
+        jobs.add(ConfigTestHelpers.randomRollupJobConfig(random(), "bar"));
         RollupIndexCaps caps = new RollupIndexCaps(ESTestCase.randomAlphaOfLength(10), jobs);
         assertTrue(caps.hasCaps());
 
@@ -45,8 +45,8 @@ public void testGetAllJobs() {
 
     public void testFilterGetJobs() {
         List<RollupJobConfig> jobs = new ArrayList<>(2);
-        jobs.add(ConfigTestHelpers.getRollupJob("foo").setIndexPattern("foo_index_pattern").build());
-        jobs.add(ConfigTestHelpers.getRollupJob("bar").build());
+        jobs.add(ConfigTestHelpers.randomRollupJobConfig(random(), "foo", "foo_index_pattern"));
+        jobs.add(ConfigTestHelpers.randomRollupJobConfig(random(), "bar"));
         RollupIndexCaps caps = new RollupIndexCaps(ESTestCase.randomAlphaOfLength(10), jobs);
         assertTrue(caps.hasCaps());
 
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
index 60c80a065cb66..069e23e4093de 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
@@ -16,9 +16,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.MockBigArrays;
-import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
@@ -30,8 +27,6 @@
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.Aggregations;
@@ -52,12 +47,15 @@
 import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.RollupField;
 import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
+import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
 import org.elasticsearch.xpack.rollup.Rollup;
 import org.hamcrest.core.IsEqual;
 import org.joda.time.DateTimeZone;
+import org.junit.Before;
 import org.mockito.Mockito;
 
 import java.io.IOException;
@@ -70,6 +68,9 @@
 import java.util.Map;
 import java.util.Set;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singleton;
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomHistogramGroupConfig;
 import static org.elasticsearch.xpack.core.rollup.RollupField.COUNT_FIELD;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -79,10 +80,12 @@
 public class SearchActionTests extends ESTestCase {
 
     private NamedWriteableRegistry namedWriteableRegistry;
+
+    @Before
     public void setUp() throws Exception {
         super.setUp();
-        IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        IndicesModule indicesModule = new IndicesModule(emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
         entries.addAll(indicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
@@ -116,11 +119,9 @@ public void testBadQuery() {
     }
 
     public void testRange() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(config);
         Set<RollupJobCaps> caps = new HashSet<>();
         caps.add(cap);
         QueryBuilder rewritten = TransportRollupSearchAction.rewriteQuery(new RangeQueryBuilder("foo").gt(1).timeZone("UTC"), caps);
@@ -129,11 +130,9 @@ public void testRange() {
     }
 
     public void testRangeNullTimeZone() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"), null, null));
+        final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(config);
         Set<RollupJobCaps> caps = new HashSet<>();
         caps.add(cap);
         QueryBuilder rewritten = TransportRollupSearchAction.rewriteQuery(new RangeQueryBuilder("foo").gt(1), caps);
@@ -142,11 +141,9 @@ public void testRangeNullTimeZone() {
     }
 
     public void testRangeWrongTZ() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"), null, "UTC"));
+        final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(config);
         Set<RollupJobCaps> caps = new HashSet<>();
         caps.add(cap);
         Exception e = expectThrows(IllegalArgumentException.class,
@@ -156,11 +153,10 @@ public void testRangeWrongTZ() {
     }
 
     public void testTermQuery() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("foo")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final TermsGroupConfig terms = new TermsGroupConfig("foo");
+        final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("boo", new DateHistogramInterval("1h")), null, terms);
+        final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(config);
         Set<RollupJobCaps> caps = new HashSet<>();
         caps.add(cap);
         QueryBuilder rewritten = TransportRollupSearchAction.rewriteQuery(new TermQueryBuilder("foo", "bar"), caps);
@@ -169,16 +165,14 @@ public void testTermQuery() {
     }
 
     public void testTermsQuery() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("foo")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final TermsGroupConfig terms = new TermsGroupConfig("foo");
+        final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("boo", new DateHistogramInterval("1h")), null, terms);
+        final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(config);
         Set<RollupJobCaps> caps = new HashSet<>();
         caps.add(cap);
         QueryBuilder original = new TermsQueryBuilder("foo", Arrays.asList("bar", "baz"));
-        QueryBuilder rewritten =
-            TransportRollupSearchAction.rewriteQuery(original, caps);
+        QueryBuilder rewritten = TransportRollupSearchAction.rewriteQuery(original, caps);
         assertThat(rewritten, instanceOf(TermsQueryBuilder.class));
         assertNotSame(rewritten, original);
         assertThat(((TermsQueryBuilder)rewritten).fieldName(), equalTo("foo.terms.value"));
@@ -186,11 +180,9 @@ public void testTermsQuery() {
     }
 
     public void testCompounds() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(config);
         Set<RollupJobCaps> caps = new HashSet<>();
         caps.add(cap);
 
@@ -202,11 +194,9 @@ public void testCompounds() {
     }
 
     public void testMatchAll() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(config);
         Set<RollupJobCaps> caps = new HashSet<>();
         caps.add(cap);
         QueryBuilder rewritten = TransportRollupSearchAction.rewriteQuery(new MatchAllQueryBuilder(), caps);
@@ -214,12 +204,10 @@ public void testMatchAll() {
     }
 
     public void testAmbiguousResolution() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        group.setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("foo")).build()).build();
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
+        final TermsGroupConfig terms = new TermsGroupConfig("foo");
+        final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")), null, terms);
+        final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(config);
         Set<RollupJobCaps> caps = new HashSet<>();
         caps.add(cap);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
@@ -365,12 +353,10 @@ public void testLiveOnlyCreateMSearch() {
     }
 
     public void testGood() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
-        Set<RollupJobCaps> caps = singletonSet(cap);
+        final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")));
+        final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(config);
+        Set<RollupJobCaps> caps = singleton(cap);
 
         String[] normalIndices = new String[]{ESTestCase.randomAlphaOfLength(10)};
         String[] rollupIndices = new String[]{ESTestCase.randomAlphaOfLength(10)};
@@ -383,7 +369,7 @@ public void testGood() {
         source.query(getQueryBuilder(1));
         source.size(0);
         source.aggregation(new DateHistogramAggregationBuilder("foo").field("foo")
-                .dateHistogramInterval(job.getGroupConfig().getDateHisto().getInterval()));
+                .dateHistogramInterval(config.getGroupConfig().getDateHistogram().getInterval()));
         SearchRequest request = new SearchRequest(combinedIndices, source);
 
         MultiSearchRequest msearch = TransportRollupSearchAction.createMSearchRequest(request, namedWriteableRegistry, ctx);
@@ -411,16 +397,10 @@ public void testGoodButNullQuery() {
         source.aggregation(new DateHistogramAggregationBuilder("foo").field("foo").dateHistogramInterval(new DateHistogramInterval("1d")));
         SearchRequest request = new SearchRequest(combinedIndices, source);
 
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo")
-                .setGroupConfig(ConfigTestHelpers.getGroupConfig()
-                        .setDateHisto(new DateHistoGroupConfig.Builder()
-                                .setInterval(new DateHistogramInterval("1d"))
-                                .setField("foo")
-                                .setTimeZone(DateTimeZone.UTC)
-                                .build())
-                        .build())
-                .build();
-        Set<RollupJobCaps> caps = singletonSet(new RollupJobCaps(job));
+        final GroupConfig groupConfig =
+            new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID()));
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        Set<RollupJobCaps> caps = singleton(new RollupJobCaps(job));
 
         TransportRollupSearchAction.RollupSearchContext ctx
                 = new TransportRollupSearchAction.RollupSearchContext(normalIndices, rollupIndices, caps);
@@ -440,20 +420,15 @@ public void testGoodButNullQuery() {
     }
 
     public void testTwoMatchingJobs() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build())
-                .setHisto(null)
-                .setTerms(null);
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
-
-        RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex());
-        job2.setGroupConfig(group.build());
+        final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")), null, null);
+        final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
 
         // so that the jobs aren't exactly equal
-        job2.setMetricsConfig(Collections.singletonList(ConfigTestHelpers.getMetricConfig().build()));
-        RollupJobCaps cap2 = new RollupJobCaps(job2.build());
+        final List<MetricConfig> metricConfigs = ConfigTestHelpers.randomMetricsConfigs(random());
+        final RollupJobConfig job2 =
+            new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10,  groupConfig, metricConfigs, null);
+        RollupJobCaps cap2 = new RollupJobCaps(job2);
 
         Set<RollupJobCaps> caps = new HashSet<>(2);
         caps.add(cap);
@@ -470,7 +445,7 @@ public void testTwoMatchingJobs() {
         source.query(getQueryBuilder(1));
         source.size(0);
         source.aggregation(new DateHistogramAggregationBuilder("foo").field("foo")
-                .dateHistogramInterval(job.getGroupConfig().getDateHisto().getInterval()));
+                .dateHistogramInterval(job.getGroupConfig().getDateHistogram().getInterval()));
         SearchRequest request = new SearchRequest(combinedIndices, source);
 
         MultiSearchRequest msearch = TransportRollupSearchAction.createMSearchRequest(request, namedWriteableRegistry, ctx);
@@ -490,21 +465,17 @@ public void testTwoMatchingJobs() {
     }
 
     public void testTwoMatchingJobsOneBetter() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build())
-            .setHisto(null)
-            .setTerms(null);
-        job.setGroupConfig(group.build());
-        RollupJobCaps cap = new RollupJobCaps(job.build());
-
-        RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex());
-        GroupConfig.Builder group2 = ConfigTestHelpers.getGroupConfig();
-        group2.setDateHisto(group.getDateHisto())
-                .setHisto(ConfigTestHelpers.getHisto().build())
-                .setTerms(null);
-        job2.setGroupConfig(group2.build());
-        RollupJobCaps cap2 = new RollupJobCaps(job2.build());
+        final GroupConfig groupConfig =
+            new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")), null, null);
+        final RollupJobConfig job =
+            new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10,  groupConfig, emptyList(), null);
+        RollupJobCaps cap = new RollupJobCaps(job);
+
+        final GroupConfig groupConfig2 =
+            new GroupConfig(groupConfig.getDateHistogram(), randomHistogramGroupConfig(random()), null);
+        final RollupJobConfig job2 =
+            new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10,  groupConfig2, emptyList(), null);
+        RollupJobCaps cap2 = new RollupJobCaps(job2);
 
         Set<RollupJobCaps> caps = new HashSet<>(2);
         caps.add(cap);
@@ -521,7 +492,7 @@ public void testTwoMatchingJobsOneBetter() {
         source.query(getQueryBuilder(1));
         source.size(0);
         source.aggregation(new DateHistogramAggregationBuilder("foo").field("foo")
-                .dateHistogramInterval(job.getGroupConfig().getDateHisto().getInterval()));
+                .dateHistogramInterval(job.getGroupConfig().getDateHistogram().getInterval()));
         SearchRequest request = new SearchRequest(combinedIndices, source);
 
         MultiSearchRequest msearch = TransportRollupSearchAction.createMSearchRequest(request, namedWriteableRegistry, ctx);
@@ -543,7 +514,8 @@ public void testTwoMatchingJobsOneBetter() {
         BoolQueryBuilder bool1 = new BoolQueryBuilder()
                 .must(TransportRollupSearchAction.rewriteQuery(request.source().query(), caps))
                 .filter(new TermQueryBuilder(RollupField.formatMetaField(RollupField.ID.getPreferredName()), "foo"))
-                .filter(new TermQueryBuilder(RollupField.formatMetaField(RollupField.VERSION_FIELD), Rollup.ROLLUP_VERSION));
+                .filter(new TermsQueryBuilder(RollupField.formatMetaField(RollupField.VERSION_FIELD),
+                    new long[]{Rollup.ROLLUP_VERSION_V1, Rollup.ROLLUP_VERSION_V2}));
         assertThat(msearch.requests().get(1).source().query(), equalTo(bool1));
     }
 
@@ -588,7 +560,7 @@ public void testMatchingIndexInMetadata() throws IOException {
         String[] indices = new String[]{"foo"};
 
         String jobName = randomAlphaOfLength(5);
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob(jobName).build();
+        RollupJobConfig job = ConfigTestHelpers.randomRollupJobConfig(random(), jobName);
 
         MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
                 Collections.singletonMap(RollupField.TYPE_NAME,
@@ -632,7 +604,7 @@ public void testRollupOnly() throws IOException {
         String[] indices = new String[]{"foo"};
 
         String jobName = randomAlphaOfLength(5);
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob(jobName).build();
+        RollupJobConfig job = ConfigTestHelpers.randomRollupJobConfig(random(), jobName);
 
         MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
                 Collections.singletonMap(RollupField.TYPE_NAME,
@@ -696,7 +668,7 @@ public void testTooManyRollups() throws IOException {
         String[] indices = new String[]{"foo", "bar"};
 
         String jobName = randomAlphaOfLength(5);
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob(jobName).build();
+        RollupJobConfig job = ConfigTestHelpers.randomRollupJobConfig(random(), jobName);
 
         MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
                 Collections.singletonMap(RollupField.TYPE_NAME,
@@ -731,7 +703,7 @@ public void testBoth() throws IOException {
         String[] indices = new String[]{"foo", "bar"};
 
         String jobName = randomAlphaOfLength(5);
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob(jobName).build();
+        RollupJobConfig job = ConfigTestHelpers.randomRollupJobConfig(random(), jobName);
 
         MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
                 Collections.singletonMap(RollupField.TYPE_NAME,
@@ -761,7 +733,7 @@ public void testBoth() throws IOException {
         SearchResponse protoResponse = mock(SearchResponse.class);
         when(protoResponse.getTook()).thenReturn(new TimeValue(100));
         List<InternalAggregation> protoAggTree = new ArrayList<>(1);
-        InternalAvg internalAvg = new InternalAvg("foo", 10, 2, DocValueFormat.RAW, Collections.emptyList(), null);
+        InternalAvg internalAvg = new InternalAvg("foo", 10, 2, DocValueFormat.RAW, emptyList(), null);
         protoAggTree.add(internalAvg);
         Aggregations protoMockAggs = new InternalAggregations(protoAggTree);
         when(protoResponse.getAggregations()).thenReturn(protoMockAggs);
@@ -801,14 +773,9 @@ public void testBoth() throws IOException {
 
         MultiSearchResponse msearchResponse
                 = new MultiSearchResponse(new MultiSearchResponse.Item[]{unrolledResponse, rolledResponse}, 123);
-
-        BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
-        ScriptService scriptService = mock(ScriptService.class);
-
         SearchResponse response = TransportRollupSearchAction.processResponses(separateIndices, msearchResponse,
                 mock(InternalAggregation.ReduceContext.class));
 
-
         assertNotNull(response);
         Aggregations responseAggs = response.getAggregations();
         assertNotNull(responseAggs);
@@ -816,10 +783,4 @@ public void testBoth() throws IOException {
         assertThat(avg.getValue(), IsEqual.equalTo(5.0));
 
     }
-
-    private Set<RollupJobCaps> singletonSet(RollupJobCaps cap) {
-        Set<RollupJobCaps> caps = new HashSet<>();
-        caps.add(cap);
-        return caps;
-    }
 }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportTaskHelperTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportTaskHelperTests.java
index b27953e4e0a62..a156585b609a7 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportTaskHelperTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportTaskHelperTests.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
 import org.elasticsearch.xpack.rollup.job.RollupJobTask;
 
@@ -18,6 +17,7 @@
 import java.util.Map;
 import java.util.function.Consumer;
 
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomRollupJobConfig;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -25,7 +25,7 @@
 public class TransportTaskHelperTests extends ESTestCase {
 
     public void testProcessRequestOneMatching() {
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo").build();
+        RollupJobConfig job = randomRollupJobConfig(random(), "foo");
         TaskManager taskManager = mock(TaskManager.class);
         RollupJobTask task = mock(RollupJobTask.class);
         when(task.getDescription()).thenReturn("rollup_foo");
@@ -58,13 +58,13 @@ public void testProcessRequestMultipleMatching() {
         Map<Long, Task> tasks = getRandomTasks();
         when(taskManager.getTasks()).thenReturn(tasks);
 
-        RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo").build();
+        RollupJobConfig job = randomRollupJobConfig(random(), "foo");
         RollupJobTask task = mock(RollupJobTask.class);
         when(task.getDescription()).thenReturn("rollup_foo");
         when(task.getConfig()).thenReturn(job);
         tasks.put(1L, task);
 
-        RollupJobConfig job2 = ConfigTestHelpers.getRollupJob("foo").build();
+        RollupJobConfig job2 = randomRollupJobConfig(random(), "foo");
         RollupJobTask task2 = mock(RollupJobTask.class);
         when(task2.getDescription()).thenReturn("rollup_foo");
         when(task2.getConfig()).thenReturn(job2);
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java
index e465c7883cfdd..86891eda669fa 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java
@@ -5,249 +5,119 @@
  */
 package org.elasticsearch.xpack.rollup.config;
 
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
-import org.elasticsearch.xpack.core.rollup.job.HistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.HistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
-import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
 import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.joda.time.DateTimeZone;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomHistogramGroupConfig;
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomRollupJobConfig;
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomTermsGroupConfig;
 import static org.hamcrest.Matchers.equalTo;
-
+//TODO split this into dedicated unit test classes (one for each config object)
 public class ConfigTests extends ESTestCase {
 
     public void testEmptyField() {
-        MetricConfig.Builder config = ConfigTestHelpers.getMetricConfig();
-        config.setField(null);
-        Exception e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [field] must be a non-null, non-empty string."));
-
-        config.setField("");
-        e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [field] must be a non-null, non-empty string."));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new MetricConfig(null, singletonList("max")));
+        assertThat(e.getMessage(), equalTo("Field must be a non-null, non-empty string"));
+
+        e = expectThrows(IllegalArgumentException.class, () -> new MetricConfig("", singletonList("max")));
+        assertThat(e.getMessage(), equalTo("Field must be a non-null, non-empty string"));
     }
 
     public void testEmptyMetrics() {
-        MetricConfig.Builder config = ConfigTestHelpers.getMetricConfig();
-        config.setMetrics(null);
-        Exception e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [metrics] must be a non-null, non-empty array of strings."));
-
-        config.setMetrics(Collections.emptyList());
-        e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [metrics] must be a non-null, non-empty array of strings."));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new MetricConfig("foo", emptyList()));
+        assertThat(e.getMessage(), equalTo("Metrics must be a non-null, non-empty array of strings"));
+
+        e = expectThrows(IllegalArgumentException.class, () -> new MetricConfig("foo", null));
+        assertThat(e.getMessage(), equalTo("Metrics must be a non-null, non-empty array of strings"));
     }
 
     public void testEmptyGroup() {
-        GroupConfig.Builder groupConfig = ConfigTestHelpers.getGroupConfig();
-        groupConfig.setDateHisto(null);
-        groupConfig.setTerms(null);
-        groupConfig.setHisto(null);
-
-        Exception e = expectThrows(IllegalArgumentException.class, groupConfig::build);
-        assertThat(e.getMessage(), equalTo("A date_histogram group is mandatory"));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new GroupConfig(null, null, null));
+        assertThat(e.getMessage(), equalTo("Date histogram must not be null"));
     }
 
     public void testNoDateHisto() {
-        GroupConfig.Builder groupConfig = new GroupConfig.Builder();
-        groupConfig.setTerms(ConfigTestHelpers.getTerms().build());
-        groupConfig.setHisto(ConfigTestHelpers.getHisto().build());
-
-        Exception e = expectThrows(IllegalArgumentException.class, groupConfig::build);
-        assertThat(e.getMessage(), equalTo("A date_histogram group is mandatory"));
+        Exception e = expectThrows(IllegalArgumentException.class,
+            () -> new GroupConfig(null, randomHistogramGroupConfig(random()), randomTermsGroupConfig(random())));
+        assertThat(e.getMessage(), equalTo("Date histogram must not be null"));
     }
 
-    public void testEmptyGroupAndMetrics() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        job.setGroupConfig(null);
-        job.setMetricsConfig(null);
-
-        Exception e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("At least one grouping or metric must be configured."));
-    }
-
-    public void testEmptyJobID() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob(null);
-        Exception e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("An ID is mandatory."));
-
-        job = ConfigTestHelpers.getRollupJob("");
-        e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("An ID is mandatory."));
-
-        job.setId("");
-        e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("An ID is mandatory."));
-
-        job.setId(null);
-        e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("An ID is mandatory."));
-    }
-
-    public void testEmptyCron() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        job.setCron("");
-        Exception e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("A cron schedule is mandatory."));
-
-        job.setCron(null);
-        e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("A cron schedule is mandatory."));
-    }
-
-    public void testBadCron() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        job.setCron("0 * * *");
-        Exception e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("invalid cron expression [0 * * *]"));
-    }
-
-    public void testEmptyIndexPattern() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        job.setIndexPattern("");
-        Exception e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("An index pattern is mandatory."));
-
-        job.setIndexPattern(null);
-        e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("An index pattern is mandatory."));
-    }
-
-    public void testMatchAllIndexPattern() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        job.setIndexPattern("*");
-        Exception e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("Index pattern must not match all indices (as it would match it's own rollup index"));
-    }
-
-    public void testMatchOwnRollupPatternPrefix() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        job.setIndexPattern("foo-*");
-        job.setRollupIndex("foo-rollup");
-        Exception e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed."));
-    }
-
-    public void testMatchOwnRollupPatternSuffix() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        job.setIndexPattern("*-rollup");
-        job.setRollupIndex("foo-rollup");
-        Exception e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed."));
-    }
+    public void testEmptyDateHistoField() {
+        Exception e = expectThrows(IllegalArgumentException.class,
+            () -> new DateHistogramGroupConfig(null, DateHistogramInterval.HOUR));
+        assertThat(e.getMessage(), equalTo("Field must be a non-null, non-empty string"));
 
-    public void testIndexPatternIdenticalToRollup() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        job.setIndexPattern("foo");
-        job.setRollupIndex("foo");
-        Exception e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("Rollup index may not be the same as the index pattern."));
+        e = expectThrows(IllegalArgumentException.class, () -> new DateHistogramGroupConfig("", DateHistogramInterval.HOUR));
+        assertThat(e.getMessage(), equalTo("Field must be a non-null, non-empty string"));
     }
 
-    public void testEmptyRollupIndex() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        job.setRollupIndex("");
-        Exception e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("A rollup index name is mandatory."));
-
-        job.setRollupIndex(null);
-        e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("A rollup index name is mandatory."));
+    public void testEmptyDateHistoInterval() {
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new DateHistogramGroupConfig("foo", null));
+        assertThat(e.getMessage(), equalTo("Interval must be non-null"));
     }
 
-    public void testBadSize() {
-        RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
-        job.setPageSize(-1);
-        Exception e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("Parameter [page_size] is mandatory and  must be a positive long."));
-
-        job.setPageSize(0);
-        e = expectThrows(IllegalArgumentException.class, job::build);
-        assertThat(e.getMessage(), equalTo("Parameter [page_size] is mandatory and  must be a positive long."));
+    public void testNullTimeZone() {
+        DateHistogramGroupConfig config = new DateHistogramGroupConfig("foo", DateHistogramInterval.HOUR, null, null);
+        assertThat(config.getTimeZone(), equalTo(DateTimeZone.UTC.getID()));
     }
 
-    public void testEmptyDateHistoField() {
-        DateHistoGroupConfig.Builder config = ConfigTestHelpers.getDateHisto();
-        config.setField(null);
-        Exception e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [field] is mandatory."));
-
-        config.setField("");
-        e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [field] is mandatory."));
+    public void testEmptyTimeZone() {
+        DateHistogramGroupConfig config = new DateHistogramGroupConfig("foo", DateHistogramInterval.HOUR, null, "");
+        assertThat(config.getTimeZone(), equalTo(DateTimeZone.UTC.getID()));
     }
 
-    public void testEmptyDateHistoInterval() {
-        DateHistoGroupConfig.Builder config = ConfigTestHelpers.getDateHisto();
-        config.setField("foo");
-        config.setInterval(null);
-        Exception e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [interval] is mandatory."));
+    public void testDefaultTimeZone() {
+        DateHistogramGroupConfig config = new DateHistogramGroupConfig("foo", DateHistogramInterval.HOUR);
+        assertThat(config.getTimeZone(), equalTo(DateTimeZone.UTC.getID()));
     }
 
-    public void testNullTimeZone() {
-        DateHistoGroupConfig.Builder config = ConfigTestHelpers.getDateHisto();
-        config.setTimeZone(null);
-        DateHistoGroupConfig finalConfig = config.build();
-        assertThat(finalConfig.getTimeZone(), equalTo(DateTimeZone.UTC));
+    public void testUnkownTimeZone() {
+        Exception e = expectThrows(IllegalArgumentException.class,
+            () -> new DateHistogramGroupConfig("foo", DateHistogramInterval.HOUR, null, "FOO"));
+        assertThat(e.getMessage(), equalTo("The datetime zone id 'FOO' is not recognised"));
     }
 
     public void testEmptyHistoField() {
-        HistoGroupConfig.Builder config = ConfigTestHelpers.getHisto();
-        config.setFields(null);
-        Exception e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [fields] must have at least one value."));
-
-        config.setFields(Collections.emptyList());
-        e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [fields] must have at least one value."));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new HistogramGroupConfig(1L, (String[]) null));
+        assertThat(e.getMessage(), equalTo("Fields must have at least one value"));
+
+        e = expectThrows(IllegalArgumentException.class, () -> new HistogramGroupConfig(1L, new String[0]));
+        assertThat(e.getMessage(), equalTo("Fields must have at least one value"));
     }
 
     public void testBadHistoIntervals() {
-        HistoGroupConfig.Builder config = new HistoGroupConfig.Builder().setFields(Collections.singletonList("foo"));
-        Exception e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [interval] must be a positive long."));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new HistogramGroupConfig(0L, "foo", "bar"));
+        assertThat(e.getMessage(), equalTo("Interval must be a positive long"));
 
-        config.setInterval(-1);
-        e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [interval] must be a positive long."));
+        e = expectThrows(IllegalArgumentException.class, () -> new HistogramGroupConfig(-1L, "foo", "bar"));
+        assertThat(e.getMessage(), equalTo("Interval must be a positive long"));
     }
 
     public void testEmptyTermsField() {
-        TermsGroupConfig.Builder config = ConfigTestHelpers.getTerms();
-        config.setFields(null);
-        Exception e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [fields] must have at least one value."));
-
-        config.setFields(Collections.emptyList());
-        e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [fields] must have at least one value."));
+        final String[] fields = randomBoolean() ? new String[0] : null;
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new TermsGroupConfig(fields));
+        assertThat(e.getMessage(), equalTo("Fields must have at least one value"));
     }
 
     public void testNoHeadersInJSON() {
         Map<String, String> headers = new HashMap<>(1);
         headers.put("es-security-runas-user", "foo");
         headers.put("_xpack_security_authentication", "bar");
-        RollupJobConfig config = ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build();
-        RollupJob job = new RollupJob(config, headers);
+        RollupJob job = new RollupJob(randomRollupJobConfig(random()), headers);
         String json = job.toString();
         assertFalse(json.contains("authentication"));
         assertFalse(json.contains("security"));
     }
-
-    public void testUnsupportedMetric() {
-        MetricConfig.Builder config = ConfigTestHelpers.getMetricConfig();
-        config.setMetrics(Arrays.asList("max","foo"));
-        Exception e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Unsupported metric [foo].  Supported metrics include: [max, min, sum, avg, value_count]"));
-    }
 }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
index 07ad0af7f1c38..e8c66f7e8c118 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
@@ -10,6 +10,7 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -34,10 +35,10 @@
 import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.RollupField;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.HistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobStats;
 import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
@@ -52,6 +53,10 @@
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomDateHistogramGroupConfig;
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomGroupConfig;
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomHistogramGroupConfig;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -94,16 +99,10 @@ public void testMissingFields() throws IOException {
 
         // Setup the composite agg
         //TODO swap this over to DateHistoConfig.Builder once DateInterval is in
-        DateHistoGroupConfig dateHistoGroupConfig = new DateHistoGroupConfig.Builder()
-                .setField(timestampField)
-                .setInterval(DateHistogramInterval.days(1))
-                .build();
+        DateHistogramGroupConfig dateHistoGroupConfig = new DateHistogramGroupConfig(timestampField, DateHistogramInterval.DAY);
         CompositeAggregationBuilder compositeBuilder =
                 new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME, dateHistoGroupConfig.toBuilders());
-        MetricConfig metricConfig = new MetricConfig.Builder()
-                .setField("does_not_exist")
-                .setMetrics(Collections.singletonList("max"))
-                .build();
+        MetricConfig metricConfig = new MetricConfig("does_not_exist", singletonList("max"));
         metricConfig.toBuilders().forEach(compositeBuilder::subAggregation);
 
         Aggregator aggregator = createAggregator(compositeBuilder, indexSearcher, timestampFieldType, valueFieldType);
@@ -114,8 +113,8 @@ public void testMissingFields() throws IOException {
         indexReader.close();
         directory.close();
 
-        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, indexName, stats,
-                ConfigTestHelpers.getGroupConfig().build(), "foo");
+        final GroupConfig groupConfig = randomGroupConfig(random());
+        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, indexName, stats, groupConfig, "foo", randomBoolean());
 
         assertThat(docs.size(), equalTo(numDocs));
         for (IndexRequest doc : docs) {
@@ -168,9 +167,9 @@ public void testCorrectFields() throws IOException {
                 .interval(1);
 
         CompositeAggregationBuilder compositeBuilder = new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME,
-                Collections.singletonList(dateHisto));
+                singletonList(dateHisto));
 
-        MetricConfig metricConfig = new MetricConfig.Builder().setField(valueField).setMetrics(Collections.singletonList("max")).build();
+        MetricConfig metricConfig = new MetricConfig(valueField, singletonList("max"));
         metricConfig.toBuilders().forEach(compositeBuilder::subAggregation);
 
         Aggregator aggregator = createAggregator(compositeBuilder, indexSearcher, timestampFieldType, valueFieldType);
@@ -181,8 +180,8 @@ public void testCorrectFields() throws IOException {
         indexReader.close();
         directory.close();
 
-        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, indexName, stats,
-                ConfigTestHelpers.getGroupConfig().build(), "foo");
+        final GroupConfig groupConfig = randomGroupConfig(random());
+        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, indexName, stats, groupConfig, "foo", randomBoolean());
 
         assertThat(docs.size(), equalTo(numDocs));
         for (IndexRequest doc : docs) {
@@ -224,9 +223,9 @@ public void testNumericTerms() throws IOException {
         TermsValuesSourceBuilder terms
                 = new TermsValuesSourceBuilder("the_terms." + TermsAggregationBuilder.NAME).field(valueField);
         CompositeAggregationBuilder compositeBuilder = new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME,
-                Collections.singletonList(terms));
+                singletonList(terms));
 
-        MetricConfig metricConfig = new MetricConfig.Builder().setField(valueField).setMetrics(Collections.singletonList("max")).build();
+        MetricConfig metricConfig = new MetricConfig(valueField, singletonList("max"));
         metricConfig.toBuilders().forEach(compositeBuilder::subAggregation);
 
         Aggregator aggregator = createAggregator(compositeBuilder, indexSearcher, valueFieldType);
@@ -237,8 +236,8 @@ public void testNumericTerms() throws IOException {
         indexReader.close();
         directory.close();
 
-        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, indexName, stats,
-                ConfigTestHelpers.getGroupConfig().build(), "foo");
+        final GroupConfig groupConfig = randomGroupConfig(random());
+        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, indexName, stats, groupConfig, "foo", randomBoolean());
 
         assertThat(docs.size(), equalTo(numDocs));
         for (IndexRequest doc : docs) {
@@ -290,9 +289,9 @@ public void testEmptyCounts() throws IOException {
                     .dateHistogramInterval(new DateHistogramInterval("1d"));
 
         CompositeAggregationBuilder compositeBuilder = new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME,
-                Collections.singletonList(dateHisto));
+                singletonList(dateHisto));
 
-        MetricConfig metricConfig = new MetricConfig.Builder().setField("another_field").setMetrics(Arrays.asList("avg", "sum")).build();
+        MetricConfig metricConfig = new MetricConfig("another_field", Arrays.asList("avg", "sum"));
         metricConfig.toBuilders().forEach(compositeBuilder::subAggregation);
 
         Aggregator aggregator = createAggregator(compositeBuilder, indexSearcher, timestampFieldType, valueFieldType);
@@ -303,8 +302,8 @@ public void testEmptyCounts() throws IOException {
         indexReader.close();
         directory.close();
 
-        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, indexName, stats,
-                ConfigTestHelpers.getGroupConfig().build(), "foo");
+        final GroupConfig groupConfig = randomGroupConfig(random());
+        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, indexName, stats, groupConfig, "foo", randomBoolean());
 
         assertThat(docs.size(), equalTo(numDocs));
         for (IndexRequest doc : docs) {
@@ -315,7 +314,7 @@ public void testEmptyCounts() throws IOException {
         }
     }
 
-    public void testKeyOrdering() {
+    public void testKeyOrderingOldID() {
         CompositeAggregation composite = mock(CompositeAggregation.class);
 
         when(composite.getBuckets()).thenAnswer((Answer<List<CompositeAggregation.Bucket>>) invocationOnMock -> {
@@ -353,14 +352,108 @@ public void testKeyOrdering() {
             return foos;
         });
 
-        GroupConfig.Builder groupConfig = ConfigTestHelpers.getGroupConfig();
-        groupConfig.setHisto(ConfigTestHelpers.getHisto().setFields(Collections.singletonList("abc")).build());
-
-        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, "foo", new RollupJobStats(), groupConfig.build(), "foo");
+        // The content of the config don't actually matter for this test
+        // because the test is just looking at agg keys
+        GroupConfig groupConfig = new GroupConfig(randomDateHistogramGroupConfig(random()), new HistogramGroupConfig(123L, "abc"), null);
+        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, "foo", new RollupJobStats(), groupConfig, "foo", false);
         assertThat(docs.size(), equalTo(1));
         assertThat(docs.get(0).id(), equalTo("1237859798"));
     }
 
+    public void testKeyOrderingNewID() {
+        CompositeAggregation composite = mock(CompositeAggregation.class);
+
+        when(composite.getBuckets()).thenAnswer((Answer<List<CompositeAggregation.Bucket>>) invocationOnMock -> {
+            List<CompositeAggregation.Bucket> foos = new ArrayList<>();
+
+            CompositeAggregation.Bucket bucket = mock(CompositeAggregation.Bucket.class);
+            LinkedHashMap<String, Object> keys = new LinkedHashMap<>(3);
+            keys.put("foo.date_histogram", 123L);
+            keys.put("bar.terms", "baz");
+            keys.put("abc.histogram", 1.9);
+            keys = shuffleMap(keys, Collections.emptySet());
+            when(bucket.getKey()).thenReturn(keys);
+
+            List<Aggregation> list = new ArrayList<>(3);
+            InternalNumericMetricsAggregation.SingleValue mockAgg = mock(InternalNumericMetricsAggregation.SingleValue.class);
+            when(mockAgg.getName()).thenReturn("123");
+            list.add(mockAgg);
+
+            InternalNumericMetricsAggregation.SingleValue mockAgg2 = mock(InternalNumericMetricsAggregation.SingleValue.class);
+            when(mockAgg2.getName()).thenReturn("abc");
+            list.add(mockAgg2);
+
+            InternalNumericMetricsAggregation.SingleValue mockAgg3 = mock(InternalNumericMetricsAggregation.SingleValue.class);
+            when(mockAgg3.getName()).thenReturn("yay");
+            list.add(mockAgg3);
+
+            Collections.shuffle(list, random());
+
+            Aggregations aggs = new Aggregations(list);
+            when(bucket.getAggregations()).thenReturn(aggs);
+            when(bucket.getDocCount()).thenReturn(1L);
+
+            foos.add(bucket);
+
+            return foos;
+        });
+
+        GroupConfig groupConfig = new GroupConfig(randomDateHistogramGroupConfig(random()), new HistogramGroupConfig(1L, "abc"), null);
+        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, "foo", new RollupJobStats(), groupConfig, "foo", true);
+        assertThat(docs.size(), equalTo(1));
+        assertThat(docs.get(0).id(), equalTo("foo$c9LcrFqeFW92uN_Z7sv1hA"));
+    }
+
+    /*
+        A test to make sure very long keys don't break the hash
+     */
+    public void testKeyOrderingNewIDLong() {
+        CompositeAggregation composite = mock(CompositeAggregation.class);
+
+        when(composite.getBuckets()).thenAnswer((Answer<List<CompositeAggregation.Bucket>>) invocationOnMock -> {
+            List<CompositeAggregation.Bucket> foos = new ArrayList<>();
+
+            CompositeAggregation.Bucket bucket = mock(CompositeAggregation.Bucket.class);
+            LinkedHashMap<String, Object> keys = new LinkedHashMap<>(3);
+            keys.put("foo.date_histogram", 123L);
+
+            char[] charArray = new char[IndexWriter.MAX_TERM_LENGTH];
+            Arrays.fill(charArray, 'a');
+            keys.put("bar.terms", new String(charArray));
+            keys.put("abc.histogram", 1.9);
+            keys = shuffleMap(keys, Collections.emptySet());
+            when(bucket.getKey()).thenReturn(keys);
+
+            List<Aggregation> list = new ArrayList<>(3);
+            InternalNumericMetricsAggregation.SingleValue mockAgg = mock(InternalNumericMetricsAggregation.SingleValue.class);
+            when(mockAgg.getName()).thenReturn("123");
+            list.add(mockAgg);
+
+            InternalNumericMetricsAggregation.SingleValue mockAgg2 = mock(InternalNumericMetricsAggregation.SingleValue.class);
+            when(mockAgg2.getName()).thenReturn("abc");
+            list.add(mockAgg2);
+
+            InternalNumericMetricsAggregation.SingleValue mockAgg3 = mock(InternalNumericMetricsAggregation.SingleValue.class);
+            when(mockAgg3.getName()).thenReturn("yay");
+            list.add(mockAgg3);
+
+            Collections.shuffle(list, random());
+
+            Aggregations aggs = new Aggregations(list);
+            when(bucket.getAggregations()).thenReturn(aggs);
+            when(bucket.getDocCount()).thenReturn(1L);
+
+            foos.add(bucket);
+
+            return foos;
+        });
+
+        GroupConfig groupConfig = new GroupConfig(randomDateHistogramGroupConfig(random()), new HistogramGroupConfig(1, "abc"), null);
+        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, "foo", new RollupJobStats(), groupConfig, "foo", true);
+        assertThat(docs.size(), equalTo(1));
+        assertThat(docs.get(0).id(), equalTo("foo$VAFKZpyaEqYRPLyic57_qw"));
+    }
+
     public void testNullKeys() {
         CompositeAggregation composite = mock(CompositeAggregation.class);
 
@@ -382,10 +475,8 @@ public void testNullKeys() {
             return foos;
         });
 
-        GroupConfig.Builder groupConfig = ConfigTestHelpers.getGroupConfig();
-        groupConfig.setHisto(ConfigTestHelpers.getHisto().setFields(Collections.singletonList("abc")).build());
-
-        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, "foo", new RollupJobStats(), groupConfig.build(), "foo");
+        GroupConfig groupConfig = new GroupConfig(randomDateHistogramGroupConfig(random()), randomHistogramGroupConfig(random()), null);
+        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, "foo", new RollupJobStats(), groupConfig, "foo", randomBoolean());
         assertThat(docs.size(), equalTo(1));
         assertFalse(Strings.isNullOrEmpty(docs.get(0).id()));
     }
@@ -431,11 +522,11 @@ public void testMissingBuckets() throws IOException {
         metricFieldType.setName(metricField);
 
         // Setup the composite agg
-        TermsGroupConfig termsGroupConfig = new TermsGroupConfig.Builder().setFields(Collections.singletonList(valueField)).build();
+        TermsGroupConfig termsGroupConfig = new TermsGroupConfig(valueField);
         CompositeAggregationBuilder compositeBuilder = new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME,
             termsGroupConfig.toBuilders()).size(numDocs*2);
 
-        MetricConfig metricConfig = new MetricConfig.Builder().setField(metricField).setMetrics(Collections.singletonList("max")).build();
+        MetricConfig metricConfig = new MetricConfig(metricField, singletonList("max"));
         metricConfig.toBuilders().forEach(compositeBuilder::subAggregation);
 
         Aggregator aggregator = createAggregator(compositeBuilder, indexSearcher, valueFieldType, metricFieldType);
@@ -446,8 +537,8 @@ public void testMissingBuckets() throws IOException {
         indexReader.close();
         directory.close();
 
-        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, indexName, stats,
-            ConfigTestHelpers.getGroupConfig().build(), "foo");
+        final GroupConfig groupConfig = randomGroupConfig(random());
+        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, indexName, stats, groupConfig, "foo", randomBoolean());
 
         assertThat(docs.size(), equalTo(6));
         for (IndexRequest doc : docs) {
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
index f658fa574eb99..6d29ee9f9ba6d 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
@@ -47,13 +47,13 @@
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
+import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.IndexerState;
 import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.Before;
@@ -71,6 +71,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
@@ -81,6 +82,7 @@
 public class RollupIndexerIndexingTests extends AggregatorTestCase {
     private QueryShardContext queryShardContext;
     private IndexSettings settings;
+    private final boolean newIDScheme = randomBoolean();
 
     @Before
     private void setup() {
@@ -93,11 +95,8 @@ private void setup() {
     public void testSimpleDateHisto() throws Exception {
         String rollupIndex = randomAlphaOfLength(10);
         String field = "the_histo";
-        DateHistoGroupConfig dateHistoConfig = new DateHistoGroupConfig.Builder()
-                .setField(field)
-                .setInterval(new DateHistogramInterval("1ms")).build();
-        RollupJobConfig job = createJob(rollupIndex, new GroupConfig.Builder().setDateHisto(dateHistoConfig).build(),
-                Collections.emptyList());
+        DateHistogramGroupConfig dateHistoConfig = new DateHistogramGroupConfig(field, new DateHistogramInterval("1ms"));
+        RollupJobConfig job = createJob(rollupIndex, new GroupConfig(dateHistoConfig), Collections.emptyList());
         final List<Map<String, Object>> dataset = new ArrayList<>();
         dataset.addAll(
                 Arrays.asList(
@@ -113,7 +112,7 @@ public void testSimpleDateHisto() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", 3,
                             "the_histo.date_histogram.interval", "1ms",
                             "the_histo.date_histogram._count", 2,
@@ -126,7 +125,7 @@ public void testSimpleDateHisto() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", 7,
                             "the_histo.date_histogram.interval", "1ms",
                             "the_histo.date_histogram._count", 1,
@@ -140,12 +139,9 @@ public void testSimpleDateHisto() throws Exception {
     public void testDateHistoAndMetrics() throws Exception {
         String rollupIndex = randomAlphaOfLength(10);
         String field = "the_histo";
-        DateHistoGroupConfig dateHistoConfig = new DateHistoGroupConfig.Builder()
-                .setField(field)
-                .setInterval(new DateHistogramInterval("1h")).build();
-        MetricConfig config = new MetricConfig.Builder().setField("counter").setMetrics(Arrays.asList("avg", "sum", "max", "min")).build();
-        RollupJobConfig job = createJob(rollupIndex, new GroupConfig.Builder().setDateHisto(dateHistoConfig).build(),
-                Collections.singletonList(config));
+        DateHistogramGroupConfig dateHistoConfig = new DateHistogramGroupConfig(field, new DateHistogramInterval("1h"));
+        MetricConfig config = new MetricConfig("counter", Arrays.asList("avg", "sum", "max", "min"));
+        RollupJobConfig job = createJob(rollupIndex, new GroupConfig(dateHistoConfig), Collections.singletonList(config));
         final List<Map<String, Object>> dataset = new ArrayList<>();
         dataset.addAll(
                 Arrays.asList(
@@ -174,7 +170,7 @@ public void testDateHistoAndMetrics() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", asLong("2015-03-31T03:00:00"),
                             "the_histo.date_histogram.interval", "1h",
                             "the_histo.date_histogram._count", 3,
@@ -192,7 +188,7 @@ public void testDateHistoAndMetrics() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", asLong("2015-03-31T04:00:00"),
                             "the_histo.date_histogram.interval", "1h",
                             "the_histo.date_histogram._count", 3,
@@ -210,7 +206,7 @@ public void testDateHistoAndMetrics() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", asLong("2015-03-31T05:00:00"),
                             "the_histo.date_histogram.interval", "1h",
                             "the_histo.date_histogram._count", 4,
@@ -228,7 +224,7 @@ public void testDateHistoAndMetrics() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", asLong("2015-03-31T06:00:00"),
                             "the_histo.date_histogram.interval", "1h",
                             "the_histo.date_histogram._count", 3,
@@ -246,7 +242,7 @@ public void testDateHistoAndMetrics() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", asLong("2015-03-31T07:00:00"),
                             "the_histo.date_histogram.interval", "1h",
                             "the_histo.date_histogram._count", 3,
@@ -265,12 +261,9 @@ public void testDateHistoAndMetrics() throws Exception {
     public void testSimpleDateHistoWithDelay() throws Exception {
         String rollupIndex = randomAlphaOfLengthBetween(5, 10);
         String field = "the_histo";
-        DateHistoGroupConfig dateHistoConfig = new DateHistoGroupConfig.Builder()
-                .setField(field)
-                .setDelay(new DateHistogramInterval("1h"))
-                .setInterval(new DateHistogramInterval("1m")).build();
-        RollupJobConfig job = createJob(rollupIndex, new GroupConfig.Builder().setDateHisto(dateHistoConfig).build(),
-                Collections.emptyList());
+        DateHistogramGroupConfig dateHistoConfig =
+            new DateHistogramGroupConfig(field, new DateHistogramInterval("1m"), new DateHistogramInterval("1h"), null);
+        RollupJobConfig job = createJob(rollupIndex, new GroupConfig(dateHistoConfig), Collections.emptyList());
         final List<Map<String, Object>> dataset = new ArrayList<>();
         long now = System.currentTimeMillis();
         dataset.addAll(
@@ -295,7 +288,7 @@ public void testSimpleDateHistoWithDelay() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", rounding.round(now - TimeValue.timeValueHours(5).getMillis()),
                             "the_histo.date_histogram.interval", "1m",
                             "the_histo.date_histogram._count", 2,
@@ -308,7 +301,7 @@ public void testSimpleDateHistoWithDelay() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", rounding.round(now - TimeValue.timeValueMinutes(75).getMillis()),
                             "the_histo.date_histogram.interval", "1m",
                             "the_histo.date_histogram._count", 2,
@@ -321,7 +314,7 @@ public void testSimpleDateHistoWithDelay() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", rounding.round(now - TimeValue.timeValueMinutes(61).getMillis()),
                             "the_histo.date_histogram.interval", "1m",
                             "the_histo.date_histogram._count", 1,
@@ -347,15 +340,11 @@ public void testSimpleDateHistoWithTimeZone() throws Exception {
                 )
         );
 
-        DateTimeZone timeZone = DateTimeZone.forOffsetHours(-3);
+        String timeZone = DateTimeZone.forOffsetHours(-3).getID();
         String rollupIndex = randomAlphaOfLengthBetween(5, 10);
         String field = "the_histo";
-        DateHistoGroupConfig dateHistoConfig = new DateHistoGroupConfig.Builder()
-                .setField(field)
-                .setTimeZone(timeZone)
-                .setInterval(new DateHistogramInterval("1d")).build();
-        RollupJobConfig job = createJob(rollupIndex, new GroupConfig.Builder().setDateHisto(dateHistoConfig).build(),
-                Collections.emptyList());
+        DateHistogramGroupConfig dateHistoConfig =  new DateHistogramGroupConfig(field, new DateHistogramInterval("1d"), null, timeZone);
+        RollupJobConfig job = createJob(rollupIndex, new GroupConfig(dateHistoConfig), Collections.emptyList());
 
         executeTestCase(dataset, job, now, (resp) -> {
                     assertThat(resp.size(), equalTo(1));
@@ -364,7 +353,7 @@ public void testSimpleDateHistoWithTimeZone() throws Exception {
                     assertThat(request.type(), equalTo("_doc"));
                     assertThat(request.sourceAsMap(), equalTo(
                             asMap(
-                                    "_rollup.version", 1,
+                                    "_rollup.version", newIDScheme ? 2 : 1,
                                     "the_histo.date_histogram.timestamp", asLong("2015-03-31T03:00:00"),
                                     "the_histo.date_histogram.interval", "1d",
                                     "the_histo.date_histogram._count", 2,
@@ -383,7 +372,7 @@ public void testSimpleDateHistoWithTimeZone() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", asLong("2015-03-31T03:00:00"),
                             "the_histo.date_histogram.interval", "1d",
                             "the_histo.date_histogram._count", 2,
@@ -396,7 +385,7 @@ public void testSimpleDateHistoWithTimeZone() throws Exception {
             assertThat(request.type(), equalTo("_doc"));
             assertThat(request.sourceAsMap(), equalTo(
                     asMap(
-                            "_rollup.version", 1,
+                            "_rollup.version", newIDScheme ? 2 : 1,
                             "the_histo.date_histogram.timestamp", asLong("2015-04-01T03:00:00"),
                             "the_histo.date_histogram.interval", "1d",
                             "the_histo.date_histogram._count", 5,
@@ -414,12 +403,10 @@ public void testRandomizedDateHisto() throws Exception {
         String valueField = "the_avg";
 
         String timeInterval = randomIntBetween(1, 10) + randomFrom("h", "m");
-        DateHistoGroupConfig dateHistoConfig = new DateHistoGroupConfig.Builder()
-                .setField(timestampField)
-                .setInterval(new DateHistogramInterval(timeInterval)).build();
-        MetricConfig metricConfig = new MetricConfig.Builder().setField(valueField).setMetrics(Collections.singletonList("avg")).build();
-        RollupJobConfig job = createJob(rollupIndex, new GroupConfig.Builder().setDateHisto(dateHistoConfig).build(),
-                Collections.singletonList(metricConfig));
+        DateHistogramGroupConfig dateHistoConfig =
+            new DateHistogramGroupConfig(timestampField, new DateHistogramInterval(timeInterval));
+        MetricConfig metricConfig = new MetricConfig(valueField, Collections.singletonList("avg"));
+        RollupJobConfig job = createJob(rollupIndex, new GroupConfig(dateHistoConfig), Collections.singletonList(metricConfig));
 
         final List<Map<String, Object>> dataset = new ArrayList<>();
         int numDocs = randomIntBetween(1,100);
@@ -435,7 +422,7 @@ public void testRandomizedDateHisto() throws Exception {
 
                 Map<String, Object> source = ((IndexRequest) request).sourceAsMap();
 
-                assertThat(source.get("_rollup.version"), equalTo(1));
+                assertThat(source.get("_rollup.version"), equalTo(newIDScheme ? 2 : 1));
                 assertThat(source.get("ts.date_histogram.interval"), equalTo(timeInterval.toString()));
                 assertNotNull(source.get("the_avg.avg._count"));
                 assertNotNull(source.get("the_avg.avg.value"));
@@ -447,15 +434,8 @@ public void testRandomizedDateHisto() throws Exception {
     }
 
     private RollupJobConfig createJob(String rollupIndex, GroupConfig groupConfig, List<MetricConfig> metricConfigs) {
-        return new RollupJobConfig.Builder()
-                .setId(randomAlphaOfLength(10))
-                .setIndexPattern(randomAlphaOfLength(10))
-                .setRollupIndex(rollupIndex)
-                .setGroupConfig(groupConfig)
-                .setMetricsConfig(metricConfigs)
-                .setCron(ConfigTestHelpers.getCronString())
-                .setPageSize(randomIntBetween(1, 100))
-                .build();
+        return new RollupJobConfig(randomAlphaOfLength(10), randomAlphaOfLength(10), rollupIndex, ConfigTestHelpers.randomCron(),
+            randomIntBetween(1, 100), groupConfig, metricConfigs, ConfigTestHelpers.randomTimeout(random()));
     }
 
     static Map<String, Object> asMap(Object... fields) {
@@ -485,7 +465,7 @@ private void executeTestCase(List<Map<String, Object>> docs, RollupJobConfig con
         Directory dir = index(docs, fieldTypeLookup);
         IndexReader reader = DirectoryReader.open(dir);
         IndexSearcher searcher = new IndexSearcher(reader);
-        String dateHistoField = config.getGroupConfig().getDateHisto().getField();
+        String dateHistoField = config.getGroupConfig().getDateHistogram().getField();
         final ExecutorService executor = Executors.newFixedThreadPool(1);
         try {
             RollupJob job = new RollupJob(config, Collections.emptyMap());
@@ -507,14 +487,14 @@ private void executeTestCase(List<Map<String, Object>> docs, RollupJobConfig con
      */
     private Map<String, MappedFieldType> createFieldTypes(RollupJobConfig job) {
         Map<String, MappedFieldType> fieldTypes = new HashMap<>();
-        MappedFieldType fieldType = new DateFieldMapper.Builder(job.getGroupConfig().getDateHisto().getField())
+        MappedFieldType fieldType = new DateFieldMapper.Builder(job.getGroupConfig().getDateHistogram().getField())
                 .dateTimeFormatter(Joda.forPattern(randomFrom("basic_date", "date_optional_time", "epoch_second")))
                 .build(new Mapper.BuilderContext(settings.getSettings(), new ContentPath(0)))
                 .fieldType();
         fieldTypes.put(fieldType.name(), fieldType);
 
-        if (job.getGroupConfig().getHisto() != null) {
-            for (String field : job.getGroupConfig().getHisto().getFields()) {
+        if (job.getGroupConfig().getHistogram() != null) {
+            for (String field : job.getGroupConfig().getHistogram().getFields()) {
                 MappedFieldType ft = new NumberFieldMapper.Builder(field, NumberFieldMapper.NumberType.LONG)
                         .build(new Mapper.BuilderContext(settings.getSettings(), new ContentPath(0)))
                         .fieldType();
@@ -590,7 +570,7 @@ class SyncRollupIndexer extends RollupIndexer {
 
         SyncRollupIndexer(Executor executor, RollupJob job, IndexSearcher searcher,
                           MappedFieldType[] fieldTypes, MappedFieldType timestampField) {
-            super(executor, job, new AtomicReference<>(IndexerState.STARTED), null);
+            super(executor, job, new AtomicReference<>(IndexerState.STARTED), null, new AtomicBoolean(newIDScheme));
             this.searcher = searcher;
             this.fieldTypes = fieldTypes;
             this.timestampField = timestampField;
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java
index 6669622b10cd1..955dcbc2beb48 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java
@@ -20,11 +20,11 @@
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.RollupField;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.IndexerState;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
 import org.mockito.stubbing.Answer;
 
@@ -49,14 +49,16 @@
 public class RollupIndexerStateTests extends ESTestCase {
     private static class EmptyRollupIndexer extends RollupIndexer {
         EmptyRollupIndexer(Executor executor, RollupJob job, AtomicReference<IndexerState> initialState,
-                           Map<String, Object> initialPosition) {
-            super(executor, job, initialState, initialPosition);
+                           Map<String, Object> initialPosition, boolean upgraded) {
+            super(executor, job, initialState, initialPosition, new AtomicBoolean(upgraded));
         }
 
-        EmptyRollupIndexer(Executor executor, RollupJob job, IndexerState initialState, Map<String, Object> initialPosition) {
-            this(executor, job, new AtomicReference<>(initialState), initialPosition);
+        EmptyRollupIndexer(Executor executor, RollupJob job, AtomicReference<IndexerState> initialState,
+                           Map<String, Object> initialPosition) {
+            this(executor, job, initialState, initialPosition, randomBoolean());
         }
 
+
         @Override
         protected void doNextSearch(SearchRequest request, ActionListener<SearchResponse> nextPhase) {
             // TODO Should use InternalComposite constructor but it is package protected in core.
@@ -92,9 +94,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 }
             }));
             final SearchResponseSections sections = new SearchResponseSections(new SearchHits(new SearchHit[0], 0, 0),
-                    aggs, null, false, null, null, 1);
+                aggs, null, false, null, null, 1);
             final SearchResponse response = new SearchResponse(sections, null, 1, 1, 0, 0,
-                    new ShardSearchFailure[0], null);
+                new ShardSearchFailure[0], null);
             nextPhase.onResponse(response);
         }
 
@@ -127,13 +129,13 @@ private static class DelayedEmptyRollupIndexer extends EmptyRollupIndexer {
         protected CountDownLatch latch;
 
         DelayedEmptyRollupIndexer(Executor executor, RollupJob job, AtomicReference<IndexerState> initialState,
-                                  Map<String, Object> initialPosition) {
-            super(executor, job, initialState, initialPosition);
+                                  Map<String, Object> initialPosition, boolean upgraded) {
+            super(executor, job, initialState, initialPosition, upgraded);
         }
 
-        DelayedEmptyRollupIndexer(Executor executor, RollupJob job, IndexerState initialState,
+        DelayedEmptyRollupIndexer(Executor executor, RollupJob job, AtomicReference<IndexerState> initialState,
                                   Map<String, Object> initialPosition) {
-            this(executor, job, new AtomicReference<>(initialState), initialPosition);
+            super(executor, job, initialState, initialPosition, randomBoolean());
         }
 
         private CountDownLatch newLatch() {
@@ -161,7 +163,7 @@ private static class NonEmptyRollupIndexer extends RollupIndexer {
         NonEmptyRollupIndexer(Executor executor, RollupJob job, AtomicReference<IndexerState> initialState,
                               Map<String, Object> initialPosition, Function<SearchRequest, SearchResponse> searchFunction,
                               Function<BulkRequest, BulkResponse> bulkFunction, Consumer<Exception> failureConsumer) {
-            super(executor, job, initialState, initialPosition);
+            super(executor, job, initialState, initialPosition, new AtomicBoolean(randomBoolean()));
             this.searchFunction = searchFunction;
             this.bulkFunction = bulkFunction;
             this.failureConsumer = failureConsumer;
@@ -206,7 +208,7 @@ protected void onAbort() {
 
         @Override
         protected void onFailure(Exception exc) {
-           failureConsumer.accept(exc);
+            failureConsumer.accept(exc);
         }
 
         @Override
@@ -214,12 +216,12 @@ protected void onFinish() {}
     }
 
     public void testStarted() throws Exception {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
-        IndexerState state = IndexerState.STOPPED;
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final ExecutorService executor = Executors.newFixedThreadPool(1);
         try {
-            RollupIndexer indexer = new EmptyRollupIndexer(executor, job, state, null);
+            RollupIndexer indexer = new EmptyRollupIndexer(executor, job, state, null, true);
+            assertTrue(indexer.isUpgradedDocumentID());
             indexer.start();
             assertThat(indexer.getState(), equalTo(IndexerState.STARTED));
             assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis()));
@@ -233,9 +235,8 @@ public void testStarted() throws Exception {
     }
 
     public void testIndexing() throws Exception {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
-        IndexerState state = IndexerState.STOPPED;
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final ExecutorService executor = Executors.newFixedThreadPool(1);
         try {
             AtomicBoolean isFinished = new AtomicBoolean(false);
@@ -270,7 +271,7 @@ public void testStateChangeMidTrigger() throws Exception {
         // and make sure the appropriate error is thrown
         when(config.getGroupConfig()).then((Answer<GroupConfig>) invocationOnMock -> {
             state.set(IndexerState.STOPPED);
-            return ConfigTestHelpers.getGroupConfig().build();
+            return ConfigTestHelpers.randomGroupConfig(random());
         });
         RollupJob job = new RollupJob(config, Collections.emptyMap());
 
@@ -301,8 +302,7 @@ protected void onFinish() {
 
     public void testAbortDuringSearch() throws Exception {
         final AtomicBoolean aborted = new AtomicBoolean(false);
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final ExecutorService executor = Executors.newFixedThreadPool(1);
         final CountDownLatch latch = new CountDownLatch(1);
@@ -346,8 +346,7 @@ protected void onAbort() {
 
     public void testAbortAfterCompletion() throws Exception {
         final AtomicBoolean aborted = new AtomicBoolean(false);
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final ExecutorService executor = Executors.newFixedThreadPool(1);
 
@@ -404,9 +403,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                         }
                     }));
                     final SearchResponseSections sections = new SearchResponseSections(new SearchHits(new SearchHit[0], 0, 0),
-                            aggs, null, false, null, null, 1);
+                        aggs, null, false, null, null, 1);
                     final SearchResponse response = new SearchResponse(sections, null, 1, 1, 0, 0,
-                            ShardSearchFailure.EMPTY_ARRAY, null);
+                        ShardSearchFailure.EMPTY_ARRAY, null);
                     nextPhase.onResponse(response);
                 }
 
@@ -432,9 +431,8 @@ protected void doSaveState(IndexerState state, Map<String, Object> position, Run
     }
 
     public void testStopIndexing() throws Exception {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
-        IndexerState state = IndexerState.STOPPED;
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final ExecutorService executor = Executors.newFixedThreadPool(1);
         try {
             DelayedEmptyRollupIndexer indexer = new DelayedEmptyRollupIndexer(executor, job, state, null);
@@ -455,9 +453,8 @@ public void testStopIndexing() throws Exception {
     }
 
     public void testAbortIndexing() throws Exception {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
-        IndexerState state = IndexerState.STOPPED;
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final ExecutorService executor = Executors.newFixedThreadPool(1);
         try {
             final AtomicBoolean isAborted = new AtomicBoolean(false);
@@ -483,9 +480,8 @@ protected void onAbort() {
     }
 
     public void testAbortStarted() throws Exception {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
-        IndexerState state = IndexerState.STOPPED;
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final ExecutorService executor = Executors.newFixedThreadPool(1);
         try {
             final AtomicBoolean isAborted = new AtomicBoolean(false);
@@ -510,9 +506,8 @@ protected void onAbort() {
     }
 
     public void testMultipleJobTriggering() throws Exception {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
-        IndexerState state = IndexerState.STOPPED;
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final ExecutorService executor = Executors.newFixedThreadPool(1);
         try {
             final AtomicBoolean isAborted = new AtomicBoolean(false);
@@ -551,8 +546,7 @@ protected void onAbort() {
     // deal with it everyhwere
     public void testUnknownKey() throws Exception {
         AtomicBoolean isFinished = new AtomicBoolean(false);
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         Function<SearchRequest, SearchResponse> searchFunction = searchRequest -> {
             Aggregations aggs = new Aggregations(Collections.singletonList(new CompositeAggregation() {
@@ -614,9 +608,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 }
             }));
             final SearchResponseSections sections = new SearchResponseSections(new SearchHits(new SearchHit[0], 0, 0),
-                    aggs, null, false, null, null, 1);
+                aggs, null, false, null, null, 1);
             return new SearchResponse(sections, null, 1, 1, 0, 0,
-                    ShardSearchFailure.EMPTY_ARRAY, null);
+                ShardSearchFailure.EMPTY_ARRAY, null);
         };
 
         Function<BulkRequest, BulkResponse> bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100);
@@ -630,7 +624,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         try {
 
             NonEmptyRollupIndexer indexer = new NonEmptyRollupIndexer(executor, job, state, null,
-                    searchFunction, bulkFunction, failureConsumer);
+                searchFunction, bulkFunction, failureConsumer);
             final CountDownLatch latch = indexer.newLatch(1);
             indexer.start();
             assertThat(indexer.getState(), equalTo(IndexerState.STARTED));
@@ -656,8 +650,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     public void testFailureWhileStopping() throws Exception {
         AtomicBoolean isFinished = new AtomicBoolean(false);
 
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         Function<SearchRequest, SearchResponse> searchFunction = searchRequest -> {
             Aggregations aggs = new Aggregations(Collections.singletonList(new CompositeAggregation() {
@@ -720,9 +713,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 }
             }));
             final SearchResponseSections sections = new SearchResponseSections(new SearchHits(new SearchHit[0], 0, 0),
-                    aggs, null, false, null, null, 1);
+                aggs, null, false, null, null, 1);
             return new SearchResponse(sections, null, 1, 1, 0, 0,
-                    ShardSearchFailure.EMPTY_ARRAY, null);
+                ShardSearchFailure.EMPTY_ARRAY, null);
         };
 
         Function<BulkRequest, BulkResponse> bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100);
@@ -736,7 +729,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         try {
 
             NonEmptyRollupIndexer indexer = new NonEmptyRollupIndexer(executor, job, state, null,
-                    searchFunction, bulkFunction, failureConsumer);
+                searchFunction, bulkFunction, failureConsumer);
             final CountDownLatch latch = indexer.newLatch(1);
             indexer.start();
             assertThat(indexer.getState(), equalTo(IndexerState.STARTED));
@@ -759,13 +752,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     public void testSearchShardFailure() throws Exception {
         AtomicBoolean isFinished = new AtomicBoolean(false);
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         Function<SearchRequest, SearchResponse> searchFunction = searchRequest -> {
             ShardSearchFailure[] failures = new ShardSearchFailure[]{new ShardSearchFailure(new RuntimeException("failed"))};
             return new SearchResponse(null, null, 1, 1, 0, 0,
-                    failures, null);
+                failures, null);
         };
 
         Function<BulkRequest, BulkResponse> bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100);
@@ -779,7 +771,7 @@ public void testSearchShardFailure() throws Exception {
         try {
 
             NonEmptyRollupIndexer indexer = new NonEmptyRollupIndexer(executor, job, state, null,
-                    searchFunction, bulkFunction, failureConsumer);
+                searchFunction, bulkFunction, failureConsumer);
             final CountDownLatch latch = indexer.newLatch(1);
             indexer.start();
             assertThat(indexer.getState(), equalTo(IndexerState.STARTED));
@@ -803,8 +795,7 @@ public void testSearchShardFailure() throws Exception {
 
     public void testBulkFailure() throws Exception {
         AtomicBoolean isFinished = new AtomicBoolean(false);
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(),
-                Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         Function<SearchRequest, SearchResponse> searchFunction = searchRequest -> {
             Aggregations aggs = new Aggregations(Collections.singletonList(new CompositeAggregation() {
@@ -866,9 +857,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 }
             }));
             final SearchResponseSections sections = new SearchResponseSections(new SearchHits(new SearchHit[0], 0, 0),
-                    aggs, null, false, null, null, 1);
+                aggs, null, false, null, null, 1);
             return new SearchResponse(sections, null, 1, 1, 0, 0,
-                    ShardSearchFailure.EMPTY_ARRAY, null);
+                ShardSearchFailure.EMPTY_ARRAY, null);
         };
 
         Function<BulkRequest, BulkResponse> bulkFunction = bulkRequest -> {
@@ -885,7 +876,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         try {
 
             NonEmptyRollupIndexer indexer = new NonEmptyRollupIndexer(executor, job, state, null,
-                    searchFunction, bulkFunction, failureConsumer) {
+                searchFunction, bulkFunction, failureConsumer) {
                 @Override
                 protected void doNextBulk(BulkRequest request, ActionListener<BulkResponse> nextPhase) {
                     nextPhase.onFailure(new RuntimeException("failed"));
@@ -911,4 +902,4 @@ protected void doNextBulk(BulkRequest request, ActionListener<BulkResponse> next
             executor.shutdownNow();
         }
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java
index ffcae267340f8..13290f09e8eb8 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java
@@ -58,89 +58,105 @@ public static void stopThreadPool() {
     }
 
     public void testInitialStatusStopped() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
-        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, Collections.singletonMap("foo", "bar"));
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, Collections.singletonMap("foo", "bar"), randomBoolean());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         SchedulerEngine schedulerEngine = new SchedulerEngine(Clock.systemUTC());
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                status, client, schedulerEngine, pool, Collections.emptyMap());
+            status, client, schedulerEngine, pool, Collections.emptyMap());
         assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
         assertThat(((RollupJobStatus)task.getStatus()).getPosition().size(), equalTo(1));
         assertTrue(((RollupJobStatus)task.getStatus()).getPosition().containsKey("foo"));
     }
 
     public void testInitialStatusAborting() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
-        RollupJobStatus status = new RollupJobStatus(IndexerState.ABORTING, Collections.singletonMap("foo", "bar"));
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        RollupJobStatus status = new RollupJobStatus(IndexerState.ABORTING, Collections.singletonMap("foo", "bar"), randomBoolean());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         SchedulerEngine schedulerEngine = new SchedulerEngine(Clock.systemUTC());
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                status, client, schedulerEngine, pool, Collections.emptyMap());
+            status, client, schedulerEngine, pool, Collections.emptyMap());
         assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
         assertThat(((RollupJobStatus)task.getStatus()).getPosition().size(), equalTo(1));
         assertTrue(((RollupJobStatus)task.getStatus()).getPosition().containsKey("foo"));
     }
 
     public void testInitialStatusStopping() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
-        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPING, Collections.singletonMap("foo", "bar"));
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPING, Collections.singletonMap("foo", "bar"), randomBoolean());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         SchedulerEngine schedulerEngine = new SchedulerEngine(Clock.systemUTC());
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                status, client, schedulerEngine, pool, Collections.emptyMap());
+            status, client, schedulerEngine, pool, Collections.emptyMap());
         assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
         assertThat(((RollupJobStatus)task.getStatus()).getPosition().size(), equalTo(1));
         assertTrue(((RollupJobStatus)task.getStatus()).getPosition().containsKey("foo"));
     }
 
     public void testInitialStatusStarted() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
-        RollupJobStatus status = new RollupJobStatus(IndexerState.STARTED, Collections.singletonMap("foo", "bar"));
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        RollupJobStatus status = new RollupJobStatus(IndexerState.STARTED, Collections.singletonMap("foo", "bar"), randomBoolean());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         SchedulerEngine schedulerEngine = new SchedulerEngine(Clock.systemUTC());
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                status, client, schedulerEngine, pool, Collections.emptyMap());
+            status, client, schedulerEngine, pool, Collections.emptyMap());
         assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STARTED));
         assertThat(((RollupJobStatus)task.getStatus()).getPosition().size(), equalTo(1));
         assertTrue(((RollupJobStatus)task.getStatus()).getPosition().containsKey("foo"));
     }
 
-    public void testInitialStatusIndexing() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
-        RollupJobStatus status = new RollupJobStatus(IndexerState.INDEXING, Collections.singletonMap("foo", "bar"));
+    public void testInitialStatusIndexingOldID() {
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        RollupJobStatus status = new RollupJobStatus(IndexerState.INDEXING, Collections.singletonMap("foo", "bar"), false);
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         SchedulerEngine schedulerEngine = new SchedulerEngine(Clock.systemUTC());
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                status, client, schedulerEngine, pool, Collections.emptyMap());
+            status, client, schedulerEngine, pool, Collections.emptyMap());
         assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STARTED));
         assertThat(((RollupJobStatus)task.getStatus()).getPosition().size(), equalTo(1));
         assertTrue(((RollupJobStatus)task.getStatus()).getPosition().containsKey("foo"));
+        assertFalse(((RollupJobStatus) task.getStatus()).isUpgradedDocumentID());
+    }
+
+    public void testInitialStatusIndexingNewID() {
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        RollupJobStatus status = new RollupJobStatus(IndexerState.INDEXING, Collections.singletonMap("foo", "bar"), true);
+        Client client = mock(Client.class);
+        when(client.settings()).thenReturn(Settings.EMPTY);
+        SchedulerEngine schedulerEngine = new SchedulerEngine(Clock.systemUTC());
+        RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
+            status, client, schedulerEngine, pool, Collections.emptyMap());
+        assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STARTED));
+        assertThat(((RollupJobStatus)task.getStatus()).getPosition().size(), equalTo(1));
+        assertTrue(((RollupJobStatus)task.getStatus()).getPosition().containsKey("foo"));
+        assertTrue(((RollupJobStatus) task.getStatus()).isUpgradedDocumentID());
     }
 
     public void testNoInitialStatus() {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         SchedulerEngine schedulerEngine = new SchedulerEngine(Clock.systemUTC());
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                null, client, schedulerEngine, pool, Collections.emptyMap());
+            null, client, schedulerEngine, pool, Collections.emptyMap());
         assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
         assertNull(((RollupJobStatus)task.getStatus()).getPosition());
+        assertTrue(((RollupJobStatus) task.getStatus()).isUpgradedDocumentID());
     }
 
     public void testStartWhenStarted() throws InterruptedException {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
-        RollupJobStatus status = new RollupJobStatus(IndexerState.STARTED, Collections.singletonMap("foo", "bar"));
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        RollupJobStatus status = new RollupJobStatus(IndexerState.STARTED, Collections.singletonMap("foo", "bar"), randomBoolean());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         SchedulerEngine schedulerEngine = new SchedulerEngine(Clock.systemUTC());
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                status, client, schedulerEngine, pool, Collections.emptyMap());
+            status, client, schedulerEngine, pool, Collections.emptyMap());
         assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STARTED));
         assertThat(((RollupJobStatus)task.getStatus()).getPosition().size(), equalTo(1));
         assertTrue(((RollupJobStatus)task.getStatus()).getPosition().containsKey("foo"));
@@ -155,7 +171,7 @@ public void onResponse(StartRollupJobAction.Response response) {
             @Override
             public void onFailure(Exception e) {
                 assertThat(e.getMessage(), equalTo("Cannot start task for Rollup Job ["
-                        + job.getConfig().getId() + "] because state was [STARTED]"));
+                    + job.getConfig().getId() + "] because state was [STARTED]"));
                 latch.countDown();
             }
         });
@@ -163,7 +179,7 @@ public void onFailure(Exception e) {
     }
 
     public void testStartWhenStopping() throws InterruptedException {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         when(client.threadPool()).thenReturn(pool);
@@ -171,7 +187,7 @@ public void testStartWhenStopping() throws InterruptedException {
 
         AtomicInteger counter = new AtomicInteger(0);
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                null, client, schedulerEngine, pool, Collections.emptyMap()) {
+            null, client, schedulerEngine, pool, Collections.emptyMap()) {
             @Override
             public void updatePersistentTaskState(PersistentTaskState taskState,
                                                   ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> listener) {
@@ -185,7 +201,7 @@ public void updatePersistentTaskState(PersistentTaskState taskState,
                     fail("Should not have updated persistent statuse > 2 times");
                 }
                 listener.onResponse(new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1,
-                        new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
+                    new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
                 counter.incrementAndGet();
             }
         };
@@ -234,7 +250,7 @@ public void onResponse(StartRollupJobAction.Response response) {
             @Override
             public void onFailure(Exception e) {
                 assertThat(e.getMessage(), equalTo("Cannot start task for Rollup Job ["
-                        + job.getConfig().getId() + "] because state was [STOPPING]"));
+                    + job.getConfig().getId() + "] because state was [STOPPING]"));
                 latch2.countDown();
             }
         });
@@ -242,20 +258,20 @@ public void onFailure(Exception e) {
     }
 
     public void testStartWhenStopped() throws InterruptedException {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
-        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, Collections.singletonMap("foo", "bar"));
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, Collections.singletonMap("foo", "bar"), randomBoolean());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         SchedulerEngine schedulerEngine = mock(SchedulerEngine.class);
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                status, client, schedulerEngine, pool, Collections.emptyMap()) {
+            status, client, schedulerEngine, pool, Collections.emptyMap()) {
             @Override
             public void updatePersistentTaskState(PersistentTaskState taskState,
                                                   ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> listener) {
                 assertThat(taskState, instanceOf(RollupJobStatus.class));
                 assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STARTED));
                 listener.onResponse(new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1,
-                        new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
+                    new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
             }
         };
         assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
@@ -280,20 +296,20 @@ public void onFailure(Exception e) {
     }
 
     public void testTriggerUnrelated() throws InterruptedException {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
-        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, Collections.singletonMap("foo", "bar"));
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, Collections.singletonMap("foo", "bar"), randomBoolean());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         SchedulerEngine schedulerEngine = mock(SchedulerEngine.class);
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                status, client, schedulerEngine, pool, Collections.emptyMap()) {
+            status, client, schedulerEngine, pool, Collections.emptyMap()) {
             @Override
             public void updatePersistentTaskState(PersistentTaskState taskState,
                                                   ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> listener) {
                 assertThat(taskState, instanceOf(RollupJobStatus.class));
                 assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STARTED));
                 listener.onResponse(new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1,
-                        new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
+                    new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
             }
         };
         assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
@@ -321,20 +337,20 @@ public void onFailure(Exception e) {
     }
 
     public void testTrigger() throws InterruptedException {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         when(client.threadPool()).thenReturn(pool);
         SchedulerEngine schedulerEngine = mock(SchedulerEngine.class);
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                null, client, schedulerEngine, pool, Collections.emptyMap()) {
+            null, client, schedulerEngine, pool, Collections.emptyMap()) {
             @Override
             public void updatePersistentTaskState(PersistentTaskState taskState,
                                                   ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> listener) {
                 assertThat(taskState, instanceOf(RollupJobStatus.class));
                 assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STARTED));
                 listener.onResponse(new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1,
-                        new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
+                    new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
             }
         };
         assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
@@ -364,7 +380,7 @@ public void onFailure(Exception e) {
     @SuppressWarnings("unchecked")
     public void testTriggerWithoutHeaders() throws InterruptedException {
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
 
@@ -395,7 +411,7 @@ public void testTriggerWithoutHeaders() throws InterruptedException {
 
         SchedulerEngine schedulerEngine = mock(SchedulerEngine.class);
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                null, client, schedulerEngine, pool, Collections.emptyMap()) {
+            null, client, schedulerEngine, pool, Collections.emptyMap()) {
             @Override
             public void updatePersistentTaskState(PersistentTaskState taskState,
                                                   ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> listener) {
@@ -404,8 +420,12 @@ public void updatePersistentTaskState(PersistentTaskState taskState,
                     assertThat(taskState, instanceOf(RollupJobStatus.class));
                     assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STARTED));
                     listener.onResponse(new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1,
-                            new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
+                        new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
                 } else if (counterValue == 1) {
+                    // When we get here, doSaveState() was just invoked so we will have
+                    // have upgraded IDs
+                    RollupJobStatus s = (RollupJobStatus)this.getStatus();
+                    assertTrue(s.isUpgradedDocumentID());
                     finished.set(true);
                 }
 
@@ -445,7 +465,95 @@ public void testTriggerWithHeaders() throws InterruptedException {
         Map<String, String> headers = new HashMap<>(1);
         headers.put("es-security-runas-user", "foo");
         headers.put("_xpack_security_authentication", "bar");
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), headers);
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), headers);
+        Client client = mock(Client.class);
+        when(client.settings()).thenReturn(Settings.EMPTY);
+
+        AtomicBoolean started = new AtomicBoolean(false);
+        AtomicBoolean finished = new AtomicBoolean(false);
+        AtomicInteger counter = new AtomicInteger(0);
+        CountDownLatch latch = new CountDownLatch(1);
+
+        final ThreadPool threadPool = mock(ThreadPool.class);
+        when(client.threadPool()).thenReturn(threadPool);
+        when(threadPool.getThreadContext()).thenReturn(threadContext);
+        doAnswer(invocationOnMock -> {
+            assertFalse(threadContext.getHeaders().isEmpty());
+            assertThat(threadContext.getHeaders().get("es-security-runas-user"), equalTo("foo"));
+            assertThat(threadContext.getHeaders().get("_xpack_security_authentication"), equalTo("bar"));
+
+            SearchResponse r = mock(SearchResponse.class);
+            when(r.getShardFailures()).thenReturn(ShardSearchFailure.EMPTY_ARRAY);
+            CompositeAggregation compositeAgg = mock(CompositeAggregation.class);
+            when(compositeAgg.getBuckets()).thenReturn(Collections.emptyList());
+            when(compositeAgg.getName()).thenReturn(RollupField.NAME);
+            Aggregations aggs = new Aggregations(Collections.singletonList(compositeAgg));
+            when(r.getAggregations()).thenReturn(aggs);
+
+            // Wait before progressing
+            latch.await();
+
+            ((ActionListener)invocationOnMock.getArguments()[2]).onResponse(r);
+            return null;
+        }).when(client).execute(anyObject(), anyObject(), anyObject());
+
+        SchedulerEngine schedulerEngine = mock(SchedulerEngine.class);
+        RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
+            null, client, schedulerEngine, pool, Collections.emptyMap()) {
+            @Override
+            public void updatePersistentTaskState(PersistentTaskState taskState,
+                                                  ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> listener) {
+                Integer counterValue = counter.getAndIncrement();
+                if (counterValue == 0) {
+                    assertThat(taskState, instanceOf(RollupJobStatus.class));
+                    assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STARTED));
+                    listener.onResponse(new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1,
+                        new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
+                } else if (counterValue == 1) {
+                    // When we get here, doSaveState() was just invoked so we will have
+                    // have upgraded IDs
+                    RollupJobStatus s = (RollupJobStatus)this.getStatus();
+                    assertTrue(s.isUpgradedDocumentID());
+                    finished.set(true);
+                }
+
+            }
+        };
+        assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
+        assertNull(((RollupJobStatus)task.getStatus()).getPosition());
+
+        task.start(new ActionListener<StartRollupJobAction.Response>() {
+            @Override
+            public void onResponse(StartRollupJobAction.Response response) {
+                assertTrue(response.isStarted());
+                assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STARTED));
+                started.set(true);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                fail("Should not have entered onFailure");
+            }
+        });
+        ESTestCase.awaitBusy(started::get);
+
+        task.triggered(new SchedulerEngine.Event(RollupJobTask.SCHEDULE_NAME + "_" + job.getConfig().getId(), 123, 123));
+        assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.INDEXING));
+        assertThat(task.getStats().getNumInvocations(), equalTo(1L));
+        // Allow search response to return now
+        latch.countDown();
+
+        // Wait for the final persistent status to finish
+        ESTestCase.awaitBusy(finished::get);
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testSaveStateChangesIDScheme() throws InterruptedException {
+        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        Map<String, String> headers = new HashMap<>(1);
+        headers.put("es-security-runas-user", "foo");
+        headers.put("_xpack_security_authentication", "bar");
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), headers);
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
 
@@ -478,8 +586,9 @@ public void testTriggerWithHeaders() throws InterruptedException {
         }).when(client).execute(anyObject(), anyObject(), anyObject());
 
         SchedulerEngine schedulerEngine = mock(SchedulerEngine.class);
+        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, null, false);
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                null, client, schedulerEngine, pool, Collections.emptyMap()) {
+            status, client, schedulerEngine, pool, Collections.emptyMap()) {
             @Override
             public void updatePersistentTaskState(PersistentTaskState taskState,
                                                   ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> listener) {
@@ -488,8 +597,12 @@ public void updatePersistentTaskState(PersistentTaskState taskState,
                     assertThat(taskState, instanceOf(RollupJobStatus.class));
                     assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STARTED));
                     listener.onResponse(new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1,
-                            new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
+                        new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
                 } else if (counterValue == 1) {
+                    // When we get here, doSaveState() was just invoked so we will have
+                    // have upgraded IDs
+                    RollupJobStatus s = (RollupJobStatus)this.getStatus();
+                    assertTrue(s.isUpgradedDocumentID());
                     finished.set(true);
                 }
 
@@ -524,13 +637,13 @@ public void onFailure(Exception e) {
     }
 
     public void testStopWhenStopped() throws InterruptedException {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
-        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, null);
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, null, randomBoolean());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         SchedulerEngine schedulerEngine = new SchedulerEngine(Clock.systemUTC());
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                status, client, schedulerEngine, pool, Collections.emptyMap());
+            status, client, schedulerEngine, pool, Collections.emptyMap());
         assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
 
         CountDownLatch latch = new CountDownLatch(1);
@@ -543,14 +656,14 @@ public void onResponse(StopRollupJobAction.Response response) {
 
             @Override
             public void onFailure(Exception e) {
-               fail("Should not have entered onFailure");
+                fail("Should not have entered onFailure");
             }
         });
         latch.await(3, TimeUnit.SECONDS);
     }
 
     public void testStopWhenStopping() throws InterruptedException {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         when(client.threadPool()).thenReturn(pool);
@@ -558,7 +671,7 @@ public void testStopWhenStopping() throws InterruptedException {
 
         AtomicInteger counter = new AtomicInteger(0);
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                null, client, schedulerEngine, pool, Collections.emptyMap()) {
+            null, client, schedulerEngine, pool, Collections.emptyMap()) {
             @Override
             public void updatePersistentTaskState(PersistentTaskState taskState,
                                                   ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> listener) {
@@ -574,7 +687,7 @@ public void updatePersistentTaskState(PersistentTaskState taskState,
                     fail("Should not have updated persistent statuse > 3 times");
                 }
                 listener.onResponse(new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1,
-                        new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
+                    new PersistentTasksCustomMetaData.Assignment("foo", "foo")));
                 counter.incrementAndGet();
 
             }
@@ -631,8 +744,8 @@ public void onFailure(Exception e) {
     }
 
     public void testStopWhenAborting() throws InterruptedException {
-        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap());
-        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, null);
+        RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
+        RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, null, randomBoolean());
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         SchedulerEngine schedulerEngine = new SchedulerEngine(Clock.systemUTC());
@@ -644,7 +757,7 @@ public void testStopWhenAborting() throws InterruptedException {
         // just in case, we can override markAsCompleted so it's a no-op and test how stop
         // handles the situation
         RollupJobTask task = new RollupJobTask(1, "type", "action", new TaskId("node", 123), job,
-                status, client, schedulerEngine, pool, Collections.emptyMap()) {
+            status, client, schedulerEngine, pool, Collections.emptyMap()) {
             @Override
             public void markAsCompleted() {
                 latch.countDown();
@@ -663,7 +776,7 @@ public void onResponse(StopRollupJobAction.Response response) {
             @Override
             public void onFailure(Exception e) {
                 assertThat(e.getMessage(), equalTo("Cannot stop task for Rollup Job ["
-                        + job.getConfig().getId() + "] because state was [ABORTING]"));
+                    + job.getConfig().getId() + "] because state was [ABORTING]"));
                 latch.countDown();
             }
         });
diff --git a/x-pack/plugin/security/cli/build.gradle b/x-pack/plugin/security/cli/build.gradle
index 19b86dd91c78b..1a00b2a034000 100644
--- a/x-pack/plugin/security/cli/build.gradle
+++ b/x-pack/plugin/security/cli/build.gradle
@@ -18,3 +18,7 @@ dependencies {
 dependencyLicenses {
     mapping from: /bc.*/, to: 'bouncycastle'
 }
+
+if (inFipsJvm) {
+    test.enabled = false
+}
\ No newline at end of file
diff --git a/x-pack/plugin/security/licenses/log4j-slf4j-impl-2.11.1.jar.sha1 b/x-pack/plugin/security/licenses/log4j-slf4j-impl-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..6178556b31848
--- /dev/null
+++ b/x-pack/plugin/security/licenses/log4j-slf4j-impl-2.11.1.jar.sha1
@@ -0,0 +1 @@
+4b41b53a3a2d299ce381a69d165381ca19f62912
\ No newline at end of file
diff --git a/x-pack/plugin/security/licenses/log4j-slf4j-impl-2.9.1.jar.sha1 b/x-pack/plugin/security/licenses/log4j-slf4j-impl-2.9.1.jar.sha1
deleted file mode 100644
index 66119e87e211f..0000000000000
--- a/x-pack/plugin/security/licenses/log4j-slf4j-impl-2.9.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0a97a849b18b3798c4af1a2ca5b10c66cef17e3a
\ No newline at end of file
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java
index 4a2c7b97195eb..cd5a720eef1dc 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java
@@ -8,6 +8,7 @@
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.bootstrap.BootstrapContext;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.XPackSettings;
 
 
 public class FIPS140JKSKeystoreBootstrapCheck implements BootstrapCheck {
@@ -15,7 +16,7 @@ public class FIPS140JKSKeystoreBootstrapCheck implements BootstrapCheck {
     private final boolean fipsModeEnabled;
 
     FIPS140JKSKeystoreBootstrapCheck(Settings settings) {
-        this.fipsModeEnabled = Security.FIPS_MODE_ENABLED.get(settings);
+        this.fipsModeEnabled = XPackSettings.FIPS_MODE_ENABLED.get(settings);
     }
 
     /**
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheck.java
new file mode 100644
index 0000000000000..d1bce0dcdd24c
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheck.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicenseService;
+
+import java.util.EnumSet;
+
+/**
+ * A bootstrap check which enforces the licensing of FIPS
+ */
+final class FIPS140LicenseBootstrapCheck implements BootstrapCheck {
+
+    static final EnumSet<License.OperationMode> ALLOWED_LICENSE_OPERATION_MODES =
+        EnumSet.of(License.OperationMode.PLATINUM, License.OperationMode.TRIAL);
+
+    private final boolean isInFipsMode;
+
+    FIPS140LicenseBootstrapCheck(boolean isInFipsMode) {
+        this.isInFipsMode = isInFipsMode;
+    }
+
+    @Override
+    public BootstrapCheckResult check(BootstrapContext context) {
+        if (isInFipsMode) {
+            License license = LicenseService.getLicense(context.metaData);
+            if (license != null && ALLOWED_LICENSE_OPERATION_MODES.contains(license.operationMode()) == false) {
+                return BootstrapCheckResult.failure("FIPS mode is only allowed with a Platinum or Trial license");
+            }
+        }
+        return BootstrapCheckResult.success();
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
index 57e12a211a38d..751d63be4fb3f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
@@ -17,7 +17,7 @@ public class FIPS140PasswordHashingAlgorithmBootstrapCheck implements BootstrapC
     private final boolean fipsModeEnabled;
 
     FIPS140PasswordHashingAlgorithmBootstrapCheck(final Settings settings) {
-        this.fipsModeEnabled = Security.FIPS_MODE_ENABLED.get(settings);
+        this.fipsModeEnabled = XPackSettings.FIPS_MODE_ENABLED.get(settings);
     }
 
     /**
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java
index c766dd0ffaa2b..a8cb32d545e8d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.XPackSettings;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -20,7 +21,7 @@ public class FIPS140SecureSettingsBootstrapCheck implements BootstrapCheck {
     private final Environment environment;
 
     FIPS140SecureSettingsBootstrapCheck(Settings settings, Environment environment) {
-        this.fipsModeEnabled = Security.FIPS_MODE_ENABLED.get(settings);
+        this.fipsModeEnabled = XPackSettings.FIPS_MODE_ENABLED.get(settings);
         this.environment = environment;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index f4bb4b7eb3b2e..18bcfeb94a502 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -176,7 +176,6 @@
 import org.elasticsearch.xpack.security.authc.TokenService;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
-import org.elasticsearch.xpack.security.authc.kerberos.KerberosRealmBootstrapCheck;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.SecuritySearchOperationListener;
@@ -256,8 +255,6 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         DiscoveryPlugin, MapperPlugin, ExtensiblePlugin {
 
     private static final Logger logger = Loggers.getLogger(Security.class);
-    static final Setting<Boolean> FIPS_MODE_ENABLED =
-        Setting.boolSetting("xpack.security.fips_mode.enabled", false, Property.NodeScope);
 
     static final Setting<List<String>> AUDIT_OUTPUTS_SETTING =
         Setting.listSetting(SecurityField.setting("audit.outputs"),
@@ -306,8 +303,7 @@ public Security(Settings settings, final Path configPath) {
                 new PasswordHashingAlgorithmBootstrapCheck(),
                 new FIPS140SecureSettingsBootstrapCheck(settings, env),
                 new FIPS140JKSKeystoreBootstrapCheck(settings),
-                new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings),
-                new KerberosRealmBootstrapCheck(env)));
+                new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings)));
             checks.addAll(InternalRealms.getBootstrapChecks(settings, env));
             this.bootstrapChecks = Collections.unmodifiableList(checks);
             Automatons.updateMaxDeterminizedStates(settings);
@@ -595,7 +591,7 @@ public static List<Setting<?>> getSettings(boolean transportClientMode, List<Sec
         }
 
         // The following just apply in node mode
-        settingsList.add(FIPS_MODE_ENABLED);
+        settingsList.add(XPackSettings.FIPS_MODE_ENABLED);
 
         // IP Filter settings
         IPFilter.addSettings(settingsList);
@@ -1002,7 +998,8 @@ public BiConsumer<DiscoveryNode, ClusterState> getJoinValidator() {
             return new ValidateTLSOnJoin(XPackSettings.TRANSPORT_SSL_ENABLED.get(settings),
                     DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings))
                 .andThen(new ValidateUpgradedSecurityIndex())
-                .andThen(new ValidateLicenseCanBeDeserialized());
+                .andThen(new ValidateLicenseCanBeDeserialized())
+                .andThen(new ValidateLicenseForFIPS(XPackSettings.FIPS_MODE_ENABLED.get(settings)));
         }
         return null;
     }
@@ -1050,6 +1047,27 @@ public void accept(DiscoveryNode node, ClusterState state) {
         }
     }
 
+    static final class ValidateLicenseForFIPS implements BiConsumer<DiscoveryNode, ClusterState> {
+        private final boolean inFipsMode;
+
+        ValidateLicenseForFIPS(boolean inFipsMode) {
+            this.inFipsMode = inFipsMode;
+        }
+
+        @Override
+        public void accept(DiscoveryNode node, ClusterState state) {
+            if (inFipsMode) {
+                License license = LicenseService.getLicense(state.metaData());
+                if (license != null &&
+                    FIPS140LicenseBootstrapCheck.ALLOWED_LICENSE_OPERATION_MODES.contains(license.operationMode()) == false) {
+                    throw new IllegalStateException("FIPS mode cannot be used with a [" + license.operationMode() +
+                        "] license. It is only allowed with a Platinum or Trial license.");
+
+                }
+            }
+        }
+    }
+
     @Override
     public void reloadSPI(ClassLoader loader) {
         securityExtensions.addAll(SecurityExtension.loadExtensions(loader));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java
index 63931d119e0f2..3e489c69d1dbc 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.xpack.core.security.action.saml.SamlLogoutResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Realm;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.authc.TokenService;
 import org.elasticsearch.xpack.security.authc.saml.SamlNameId;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java
index 57510ce116f7d..af56ab8d4eb7d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequest;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateResponse;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 
 import java.util.function.Supplier;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
index 7e17cda75f0ab..f89745d23e317 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.xpack.core.security.action.user.GetUsersResponse;
 import org.elasticsearch.xpack.core.security.authc.esnative.ClientReservedRealm;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java
index b49984b28da08..eefaaa72b1e04 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
 import org.elasticsearch.xpack.core.security.support.Automatons;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java
index 3f19d28192583..8dcaf1a61ff47 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.transport.TransportMessage;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 
 import java.net.InetAddress;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java
index 3cd12b1a7ceb9..a9245c7653e04 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.transport.TransportMessage;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 
 import java.net.InetAddress;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java
index d8b4b4e4bc198..6d12455fdea6e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java
@@ -55,7 +55,7 @@
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.index.IndexAuditTrailField;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
 import org.elasticsearch.xpack.security.audit.AuditLevel;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
index 5da6a9eb77cdd..3a6cfa501935c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.audit.AuditLevel;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
index 85084da84648d..416acfca3abd2 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.xpack.core.security.support.Exceptions;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
index 0284ae9a05fa3..8b80c1f1d1ca9 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 
 
 /**
@@ -152,6 +153,7 @@ protected List<Realm> initRealms() throws Exception {
         Settings realmsSettings = RealmSettings.get(settings);
         Set<String> internalTypes = new HashSet<>();
         List<Realm> realms = new ArrayList<>();
+        List<String> kerberosRealmNames = new ArrayList<>();
         for (String name : realmsSettings.names()) {
             Settings realmSettings = realmsSettings.getAsSettings(name);
             String type = realmSettings.get("type");
@@ -178,6 +180,13 @@ protected List<Realm> initRealms() throws Exception {
                 }
                 internalTypes.add(type);
             }
+            if (KerberosRealmSettings.TYPE.equals(type)) {
+                kerberosRealmNames.add(name);
+                if (kerberosRealmNames.size() > 1) {
+                    throw new IllegalArgumentException("multiple realms " + kerberosRealmNames.toString() + " configured of type [" + type
+                            + "], [" + type + "] can only have one such realm configured");
+                }
+            }
             realms.add(factory.create(config));
         }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java
index 4226607f192f4..f28fe1c297f5b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java
@@ -363,9 +363,16 @@ static Logger getTerminalLogger(final Terminal terminal) {
         final Logger logger = ESLoggerFactory.getLogger(ESNativeRealmMigrateTool.class);
         Loggers.setLevel(logger, Level.ALL);
 
+        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+        final Configuration config = ctx.getConfiguration();
+
         // create appender
         final Appender appender = new AbstractAppender(ESNativeRealmMigrateTool.class.getName(), null,
-                PatternLayout.newBuilder().withPattern("%m").build()) {
+                PatternLayout.newBuilder()
+                    // Specify the configuration so log4j doesn't re-initialize
+                    .withConfiguration(config)
+                    .withPattern("%m")
+                    .build()) {
             @Override
             public void append(LogEvent event) {
                 switch (event.getLevel().getStandardLevel()) {
@@ -384,8 +391,6 @@ public void append(LogEvent event) {
         appender.start();
 
         // get the config, detach from parent, remove appenders, add custom appender
-        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
-        final Configuration config = ctx.getConfiguration();
         final LoggerConfig loggerConfig = config.getLoggerConfig(ESNativeRealmMigrateTool.class.getName());
         loggerConfig.setParent(null);
         loggerConfig.getAppenders().forEach((s, a) -> Loggers.removeAppender(logger, a));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java
index a84b76beab8bb..ffd9c3f73bcad 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
index d923a0298041b..507ed4684a14e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
@@ -48,8 +48,8 @@
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
-import org.elasticsearch.xpack.core.security.user.User.Fields;
+import org.elasticsearch.protocol.xpack.security.User;
+import org.elasticsearch.protocol.xpack.security.User.Fields;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java
index 31de03ff4330c..99c138bbb121d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.LogstashSystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore.ReservedUserInfo;
 import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java
index 3f636312f0f06..d9971ab2388f8 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java
@@ -7,7 +7,7 @@
 
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 /**
  * Like User, but includes the hashed password
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java
index e2586ea836dec..8d529897534e5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm;
 
 import java.util.Map;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java
index 15a6c2c41daed..220108b563755 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.xpack.core.security.support.NoOpLogger;
 import org.elasticsearch.xpack.core.security.support.Validation;
 import org.elasticsearch.xpack.core.security.support.Validation.Users;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.support.SecurityFiles;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
index b4a8b6aabf076..53146203ee2f1 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
@@ -19,12 +19,13 @@
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.support.CachingRealm;
 import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 import org.ietf.jgss.GSSException;
 
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.List;
@@ -87,6 +88,16 @@ public KerberosRealm(final RealmConfig config, final NativeRoleMappingStore nati
         this.kerberosTicketValidator = kerberosTicketValidator;
         this.threadPool = threadPool;
         this.keytabPath = config.env().configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(config.settings()));
+
+        if (Files.exists(keytabPath) == false) {
+            throw new IllegalArgumentException("configured service key tab file [" + keytabPath + "] does not exist");
+        }
+        if (Files.isDirectory(keytabPath)) {
+            throw new IllegalArgumentException("configured service key tab file [" + keytabPath + "] is a directory");
+        }
+        if (Files.isReadable(keytabPath) == false) {
+            throw new IllegalArgumentException("configured service key tab file [" + keytabPath + "] must have read permission");
+        }
         this.enableKerberosDebug = KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE.get(config.settings());
         this.removeRealmName = KerberosRealmSettings.SETTING_REMOVE_REALM_NAME.get(config.settings());
     }
@@ -168,12 +179,15 @@ protected String maybeRemoveRealmName(final String principalName) {
 
     private void handleException(Exception e, final ActionListener<AuthenticationResult> listener) {
         if (e instanceof LoginException) {
+            logger.debug("failed to authenticate user, service login failure", e);
             listener.onResponse(AuthenticationResult.terminate("failed to authenticate user, service login failure",
                     unauthorized(e.getLocalizedMessage(), e)));
         } else if (e instanceof GSSException) {
+            logger.debug("failed to authenticate user, gss context negotiation failure", e);
             listener.onResponse(AuthenticationResult.terminate("failed to authenticate user, gss context negotiation failure",
                     unauthorized(e.getLocalizedMessage(), e)));
         } else {
+            logger.debug("failed to authenticate user", e);
             listener.onFailure(e);
         }
     }
@@ -211,4 +225,4 @@ private void buildUser(final String username, final String outToken, final Actio
     public void lookupUser(final String username, final ActionListener<User> listener) {
         listener.onResponse(null);
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheck.java
deleted file mode 100644
index bab899a866425..0000000000000
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheck.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.authc.kerberos;
-
-import org.elasticsearch.bootstrap.BootstrapCheck;
-import org.elasticsearch.bootstrap.BootstrapContext;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.xpack.core.security.authc.RealmSettings;
-import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * This class is used to perform bootstrap checks for kerberos realm.
- * <p>
- * We use service keytabs for validating incoming kerberos tickets and is a
- * required configuration. Due to JVM wide system properties for Kerberos we
- * cannot support multiple Kerberos realms. This class adds checks for node to
- * fail if service keytab does not exist or multiple kerberos realms have been
- * configured.
- */
-public class KerberosRealmBootstrapCheck implements BootstrapCheck {
-    private final Environment env;
-
-    public KerberosRealmBootstrapCheck(final Environment env) {
-        this.env = env;
-    }
-
-    @Override
-    public BootstrapCheckResult check(final BootstrapContext context) {
-        final Map<String, Settings> realmsSettings = RealmSettings.getRealmSettings(context.settings);
-        boolean isKerberosRealmConfigured = false;
-        for (final Entry<String, Settings> entry : realmsSettings.entrySet()) {
-            final String name = entry.getKey();
-            final Settings realmSettings = entry.getValue();
-            final String type = realmSettings.get("type");
-            if (Strings.hasText(type) == false) {
-                return BootstrapCheckResult.failure("missing realm type for [" + name + "] realm");
-            }
-            if (KerberosRealmSettings.TYPE.equals(type)) {
-                if (isKerberosRealmConfigured) {
-                    return BootstrapCheckResult.failure(
-                            "multiple [" + type + "] realms are configured. [" + type + "] can only have one such realm configured");
-                }
-                isKerberosRealmConfigured = true;
-
-                final Path keytabPath = env.configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(realmSettings));
-                if (Files.exists(keytabPath) == false) {
-                    return BootstrapCheckResult.failure("configured service key tab file [" + keytabPath + "] does not exist");
-                }
-            }
-        }
-        return BootstrapCheckResult.success();
-    }
-
-    @Override
-    public boolean alwaysEnforce() {
-        return true;
-    }
-}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java
index 87749850141bf..f689bc2878903 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.LdapSessionFactorySettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapLoadBalancing;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java
index 7b9eabfd7066f..58e10a5475571 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
 import org.elasticsearch.xpack.security.authc.BytesKey;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
index cc160c8f78b33..a8f50d975e8d7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
@@ -43,7 +43,7 @@
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.ssl.SSLConfiguration;
 import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
 import org.elasticsearch.xpack.core.ssl.SSLService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java
index 68338e9bcad93..bcdbc1e1dd305 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.util.Collections;
 import java.util.Map;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
index 642bc167f7d4a..2c9bb8ce3c0ce 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
@@ -57,7 +57,7 @@
 import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackSecurityUser;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java
index 3068a3993d309..07845a131b750 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java
@@ -8,7 +8,7 @@
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.util.ArrayList;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java
index 15ac88b4d9462..6db19d8edeb1b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.util.Arrays;
 import java.util.EnumSet;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java
index b280b3a89a204..10b8e65c16824 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequest;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateResponse;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.io.IOException;
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
index 1b64b3ce2baee..7e33844e99bb1 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.rest.RestRequestFilter;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java
index 1ab80954e9b53..3e8cf26ad7d6b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xpack.core.security.action.user.GetUsersResponse;
 import org.elasticsearch.xpack.core.security.client.SecurityClient;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
index 2f0c40c1fdd16..761af81b08ec5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.action.SecurityActionMapper;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
index 8ce72be3ef9a7..df6cf33c65d8e 100644
--- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
@@ -25,6 +25,10 @@ grant {
   permission java.util.PropertyPermission "sun.security.krb5.debug","write";
   permission java.util.PropertyPermission "java.security.debug","write";
   permission java.util.PropertyPermission "sun.security.spnego.debug","write";
+
+  // needed for kerberos file permission tests to access user information
+  permission java.lang.RuntimePermission "accessUserInformation";
+  permission java.lang.RuntimePermission "getFileStoreAttributes";
 };
 
 grant codeBase "${codebase.xmlsec-2.0.8.jar}" {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java
index 03ace7fb71dc7..bb0036e9f870a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java
@@ -5,15 +5,12 @@
  */
 package org.elasticsearch.integration;
 
-import org.apache.http.Header;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.message.BasicHeader;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.client.Response;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -24,10 +21,8 @@
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 
 import java.io.IOException;
-import java.util.Collections;
 
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
 
 public class BulkUpdateTests extends SecurityIntegTestCase {
 
@@ -77,46 +72,48 @@ public void testThatBulkUpdateDoesNotLoseFields() {
 
     public void testThatBulkUpdateDoesNotLoseFieldsHttp() throws IOException {
         final String path = "/index1/type/1";
-        final Header basicAuthHeader = new BasicHeader("Authorization",
-                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                        new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+        final RequestOptions.Builder optionsBuilder = RequestOptions.DEFAULT.toBuilder();
+        optionsBuilder.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+        final RequestOptions options = optionsBuilder.build();
 
-        StringEntity body = new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON);
-        Response response = getRestClient().performRequest("PUT", path, Collections.emptyMap(), body, basicAuthHeader);
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(201));
+        Request createRequest = new Request("PUT", path);
+        createRequest.setOptions(options);
+        createRequest.setJsonEntity("{\"test\":\"test\"}");
+        getRestClient().performRequest(createRequest);
 
-        response = getRestClient().performRequest("GET", path, basicAuthHeader);
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-        assertThat(EntityUtils.toString(response.getEntity()), containsString("\"test\":\"test\""));
+        Request getRequest = new Request("GET", path);
+        getRequest.setOptions(options);
+        assertThat(EntityUtils.toString(getRestClient().performRequest(getRequest).getEntity()), containsString("\"test\":\"test\""));
 
         if (randomBoolean()) {
             flushAndRefresh();
         }
 
         //update with new field
-        body = new StringEntity("{\"doc\": {\"not test\": \"not test\"}}", ContentType.APPLICATION_JSON);
-        response = getRestClient().performRequest("POST", path + "/_update", Collections.emptyMap(), body, basicAuthHeader);
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+        Request updateRequest = new Request("POST", path + "/_update");
+        updateRequest.setOptions(options);
+        updateRequest.setJsonEntity("{\"doc\": {\"not test\": \"not test\"}}");
+        getRestClient().performRequest(updateRequest);
 
-        response = getRestClient().performRequest("GET", path, basicAuthHeader);
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-        String responseBody = EntityUtils.toString(response.getEntity());
-        assertThat(responseBody, containsString("\"test\":\"test\""));
-        assertThat(responseBody, containsString("\"not test\":\"not test\""));
+        String afterUpdate = EntityUtils.toString(getRestClient().performRequest(getRequest).getEntity());
+        assertThat(afterUpdate, containsString("\"test\":\"test\""));
+        assertThat(afterUpdate, containsString("\"not test\":\"not test\""));
 
         // this part is important. Without this, the document may be read from the translog which would bypass the bug where
         // FLS kicks in because the request can't be found and only returns meta fields
         flushAndRefresh();
 
-        body = new StringEntity("{\"update\": {\"_index\": \"index1\", \"_type\": \"type\", \"_id\": \"1\"}}\n" +
-                "{\"doc\": {\"bulk updated\":\"bulk updated\"}}\n", ContentType.APPLICATION_JSON);
-        response = getRestClient().performRequest("POST", "/_bulk", Collections.emptyMap(), body, basicAuthHeader);
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-
-        response = getRestClient().performRequest("GET", path, basicAuthHeader);
-        responseBody = EntityUtils.toString(response.getEntity());
-        assertThat(responseBody, containsString("\"test\":\"test\""));
-        assertThat(responseBody, containsString("\"not test\":\"not test\""));
-        assertThat(responseBody, containsString("\"bulk updated\":\"bulk updated\""));
+        Request bulkRequest = new Request("POST", "/_bulk");
+        bulkRequest.setOptions(options);
+        bulkRequest.setJsonEntity(
+                "{\"update\": {\"_index\": \"index1\", \"_type\": \"type\", \"_id\": \"1\"}}\n" +
+                "{\"doc\": {\"bulk updated\":\"bulk updated\"}}\n");
+        getRestClient().performRequest(bulkRequest);
+
+        String afterBulk = EntityUtils.toString(getRestClient().performRequest(getRequest).getEntity());
+        assertThat(afterBulk, containsString("\"test\":\"test\""));
+        assertThat(afterBulk, containsString("\"not test\":\"not test\""));
+        assertThat(afterBulk, containsString("\"bulk updated\":\"bulk updated\""));
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java
index 8b9e195426493..0b6321e596010 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java
@@ -5,10 +5,11 @@
  */
 package org.elasticsearch.integration;
 
-import org.apache.http.message.BasicHeader;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
@@ -21,7 +22,7 @@
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.client.SecurityClient;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.Realms;
 import org.junit.BeforeClass;
 
@@ -160,10 +161,15 @@ public void onFailure(Exception e) {
         }
 
         static void executeHttpRequest(String path, Map<String, String> params) throws Exception {
-            Response response = getRestClient().performRequest("POST", path, params,
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                                    new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))));
+            Request request = new Request("POST", path);
+            for (Map.Entry<String, String> param : params.entrySet()) {
+                request.addParameter(param.getKey(), param.getValue());
+            }
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                    new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+            request.setOptions(options);
+            Response response = getRestClient().performRequest(request);
             assertNotNull(response.getEntity());
             assertTrue(EntityUtils.toString(response.getEntity()).contains("cluster_name"));
         }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
index 2250facc37b51..8128b03a065b4 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
@@ -97,7 +97,8 @@ protected String configUsers() {
         return super.configUsers() +
             "user1:" + usersPasswdHashed + "\n" +
             "user2:" + usersPasswdHashed + "\n" +
-            "user3:" + usersPasswdHashed + "\n";
+            "user3:" + usersPasswdHashed + "\n" +
+            "user4:" + usersPasswdHashed + "\n";
     }
 
     @Override
@@ -105,7 +106,8 @@ protected String configUsersRoles() {
         return super.configUsersRoles() +
                 "role1:user1,user2,user3\n" +
                 "role2:user1,user3\n" +
-                "role3:user2,user3\n";
+                "role3:user2,user3\n" +
+                "role4:user4\n";
     }
 
     @Override
@@ -131,7 +133,14 @@ protected String configRoles() {
                 "  indices:\n" +
                 "    - names: '*'\n" +
                 "      privileges: [ ALL ]\n" +
-                "      query: '{\"term\" : {\"field2\" : \"value2\"}}'"; // <-- query defined as json in a string
+                "      query: '{\"term\" : {\"field2\" : \"value2\"}}'\n" + // <-- query defined as json in a string
+                "role4:\n" +
+                "  cluster: [ all ]\n" +
+                "  indices:\n" +
+                "    - names: '*'\n" +
+                "      privileges: [ ALL ]\n" +
+                // query that can match nested documents
+                "      query: '{\"bool\": { \"must_not\": { \"term\" : {\"field1\" : \"value2\"}}}}'";
     }
 
     @Override
@@ -869,7 +878,7 @@ public void testNestedInnerHits() throws Exception {
         refresh("test");
 
         SearchResponse response = client()
-                .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
                 .prepareSearch("test")
                 .setQuery(QueryBuilders.nestedQuery("nested_field", QueryBuilders.termQuery("nested_field.field2", "value2"),
                         ScoreMode.None).innerHit(new InnerHitBuilder()))
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java
index 825bfcd432f7c..57262822982fa 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java
@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.integration;
 
-import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
@@ -388,9 +389,12 @@ public void testUserU14() throws Exception {
 
     public void testThatUnknownUserIsRejectedProperly() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue("idonotexist", new SecureString("passwd".toCharArray()))));
+            Request request = new Request("GET", "/");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization",
+                    UsernamePasswordToken.basicAuthHeaderValue("idonotexist", new SecureString("passwd".toCharArray())));
+            request.setOptions(options);
+            getRestClient().performRequest(request);
             fail("request should have failed");
         } catch(ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
index 012050f42598c..7a35b0bc422ae 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.license;
 
-import org.apache.http.message.BasicHeader;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
@@ -15,6 +14,8 @@
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
@@ -189,31 +190,36 @@ public void testEnableDisableBehaviour() throws Exception {
     }
 
     public void testRestAuthenticationByLicenseType() throws Exception {
-        Response response = getRestClient().performRequest("GET", "/");
+        Response unauthorizedRootResponse = getRestClient().performRequest(new Request("GET", "/"));
         // the default of the licensing tests is basic
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
+        assertThat(unauthorizedRootResponse.getStatusLine().getStatusCode(), is(200));
         ResponseException e = expectThrows(ResponseException.class,
-                () -> getRestClient().performRequest("GET", "/_xpack/security/_authenticate"));
+                () -> getRestClient().performRequest(new Request("GET", "/_xpack/security/_authenticate")));
         assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
 
         // generate a new license with a mode that enables auth
         License.OperationMode mode = randomFrom(License.OperationMode.GOLD, License.OperationMode.TRIAL,
                 License.OperationMode.PLATINUM, License.OperationMode.STANDARD);
         enableLicensing(mode);
-        e = expectThrows(ResponseException.class, () -> getRestClient().performRequest("GET", "/"));
+        e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(new Request("GET", "/")));
         assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
         e = expectThrows(ResponseException.class,
-            () -> getRestClient().performRequest("GET", "/_xpack/security/_authenticate"));
+            () -> getRestClient().performRequest(new Request("GET", "/_xpack/security/_authenticate")));
         assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
 
-        final String basicAuthValue = UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()));
-        response = getRestClient().performRequest("GET", "/", new BasicHeader("Authorization", basicAuthValue));
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        response = getRestClient().performRequest("GET", "/_xpack/security/_authenticate",
-                    new BasicHeader("Authorization", basicAuthValue));
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
+        RequestOptions.Builder optionsBuilder = RequestOptions.DEFAULT.toBuilder();
+        optionsBuilder.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+        RequestOptions options = optionsBuilder.build();
+
+        Request rootRequest = new Request("GET", "/");
+        rootRequest.setOptions(options);
+        Response authorizedRootResponse = getRestClient().performRequest(rootRequest);
+        assertThat(authorizedRootResponse.getStatusLine().getStatusCode(), is(200));
+        Request authenticateRequest = new Request("GET", "/_xpack/security/_authenticate");
+        authenticateRequest.setOptions(options);
+        Response authorizedAuthenticateResponse = getRestClient().performRequest(authenticateRequest);
+        assertThat(authorizedAuthenticateResponse.getStatusLine().getStatusCode(), is(200));
     }
 
     public void testSecurityActionsByLicenseType() throws Exception {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java
index 725508327796a..af5b73d889dcf 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java
@@ -5,12 +5,8 @@
  */
 package org.elasticsearch.test;
 
-
-import org.apache.http.HttpEntity;
-import org.apache.http.entity.ContentType;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.nio.entity.NStringEntity;
-import org.elasticsearch.client.Response;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.util.set.Sets;
@@ -26,7 +22,6 @@
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.Set;
 
 /**
@@ -82,23 +77,22 @@ public void setupReservedPasswords() throws IOException {
     public void setupReservedPasswords(RestClient restClient) throws IOException {
         logger.info("setting up reserved passwords for test");
         {
-            String payload = "{\"password\": \"" + new String(reservedPassword.getChars()) + "\"}";
-            HttpEntity entity = new NStringEntity(payload, ContentType.APPLICATION_JSON);
-            BasicHeader authHeader = new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                    UsernamePasswordToken.basicAuthHeaderValue(ElasticUser.NAME, BOOTSTRAP_PASSWORD));
-            String route = "/_xpack/security/user/elastic/_password";
-            Response response = restClient.performRequest("PUT", route, Collections.emptyMap(), entity, authHeader);
-            assertEquals(response.getStatusLine().getReasonPhrase(), 200, response.getStatusLine().getStatusCode());
+            Request request = new Request("PUT", "/_xpack/security/user/elastic/_password");
+            request.setJsonEntity("{\"password\": \"" + new String(reservedPassword.getChars()) + "\"}");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(ElasticUser.NAME, BOOTSTRAP_PASSWORD));
+            request.setOptions(options);
+            restClient.performRequest(request);
         }
 
+        RequestOptions.Builder optionsBuilder = RequestOptions.DEFAULT.toBuilder();
+        optionsBuilder.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(ElasticUser.NAME, reservedPassword));
+        RequestOptions options = optionsBuilder.build();
         for (String username : Arrays.asList(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME)) {
-            String payload = "{\"password\": \"" + new String(reservedPassword.getChars()) + "\"}";
-            HttpEntity entity = new NStringEntity(payload, ContentType.APPLICATION_JSON);
-            BasicHeader authHeader = new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                    UsernamePasswordToken.basicAuthHeaderValue(ElasticUser.NAME, reservedPassword));
-            String route = "/_xpack/security/user/" + username + "/_password";
-            Response response = restClient.performRequest("PUT", route, Collections.emptyMap(), entity, authHeader);
-            assertEquals(response.getStatusLine().getReasonPhrase(), 200, response.getStatusLine().getStatusCode());
+            Request request = new Request("PUT", "/_xpack/security/user/" + username + "/_password");
+            request.setJsonEntity("{\"password\": \"" + new String(reservedPassword.getChars()) + "\"}");
+            request.setOptions(options);
+            restClient.performRequest(request);
         }
         logger.info("setting up reserved passwords finished");
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheckTests.java
new file mode 100644
index 0000000000000..a2ec8f9fb2053
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheckTests.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.TestUtils;
+import org.elasticsearch.test.ESTestCase;
+
+public class FIPS140LicenseBootstrapCheckTests extends ESTestCase {
+
+    public void testBootstrapCheck() throws Exception {
+        assertTrue(new FIPS140LicenseBootstrapCheck(false)
+            .check(new BootstrapContext(Settings.EMPTY, MetaData.EMPTY_META_DATA)).isSuccess());
+        assertTrue(new FIPS140LicenseBootstrapCheck(randomBoolean())
+            .check(new BootstrapContext(Settings.EMPTY, MetaData.EMPTY_META_DATA)).isSuccess());
+
+        License license = TestUtils.generateSignedLicense(TimeValue.timeValueHours(24));
+        MetaData.Builder builder = MetaData.builder();
+        TestUtils.putLicense(builder, license);
+        MetaData metaData = builder.build();
+        if (FIPS140LicenseBootstrapCheck.ALLOWED_LICENSE_OPERATION_MODES.contains(license.operationMode())) {
+            assertTrue(new FIPS140LicenseBootstrapCheck(true).check(new BootstrapContext(
+                Settings.builder().put("xpack.security.fips_mode.enabled", true).build(), metaData)).isSuccess());
+            assertTrue(new FIPS140LicenseBootstrapCheck(false).check(new BootstrapContext(
+                Settings.builder().put("xpack.security.fips_mode.enabled", false).build(), metaData)).isSuccess());
+        } else {
+            assertTrue(new FIPS140LicenseBootstrapCheck(false).check(new BootstrapContext(
+                Settings.builder().put("xpack.security.fips_mode.enabled", false).build(), metaData)).isSuccess());
+            assertTrue(new FIPS140LicenseBootstrapCheck(true).check(new BootstrapContext(
+                Settings.builder().put("xpack.security.fips_mode.enabled", true).build(), metaData)).isFailure());
+            assertEquals("FIPS mode is only allowed with a Platinum or Trial license",
+                new FIPS140LicenseBootstrapCheck(true).check(new BootstrapContext(
+                    Settings.builder().put("xpack.security.fips_mode.enabled", true).build(), metaData)).getMessage());
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
index b424986876aa4..8632400866a07 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
@@ -21,7 +21,7 @@ public class FIPS140PasswordHashingAlgorithmBootstrapCheckTests extends ESTestCa
     public void testPBKDF2AlgorithmIsAllowed() {
         {
             final Settings settings = Settings.builder()
-                    .put(Security.FIPS_MODE_ENABLED.getKey(), true)
+                    .put(XPackSettings.FIPS_MODE_ENABLED.getKey(), true)
                     .put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2_10000")
                     .build();
             final BootstrapCheck.BootstrapCheckResult result =
@@ -31,7 +31,7 @@ public void testPBKDF2AlgorithmIsAllowed() {
 
         {
             final Settings settings = Settings.builder()
-                    .put(Security.FIPS_MODE_ENABLED.getKey(), true)
+                    .put(XPackSettings.FIPS_MODE_ENABLED.getKey(), true)
                     .put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2")
                     .build();
             final BootstrapCheck.BootstrapCheckResult result =
@@ -49,7 +49,7 @@ public void testBCRYPTAlgorithmDependsOnFipsMode() {
     }
 
     private void runBCRYPTTest(final boolean fipsModeEnabled, final String passwordHashingAlgorithm) {
-        final Settings.Builder builder = Settings.builder().put(Security.FIPS_MODE_ENABLED.getKey(), fipsModeEnabled);
+        final Settings.Builder builder = Settings.builder().put(XPackSettings.FIPS_MODE_ENABLED.getKey(), fipsModeEnabled);
         if (passwordHashingAlgorithm != null) {
             builder.put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), passwordHashingAlgorithm);
         }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java
index e3b1cd31246a2..2b9e540f3bfe4 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.junit.Before;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java
index 368449adeedc4..d57a5d151a95f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java
@@ -5,14 +5,14 @@
  */
 package org.elasticsearch.xpack.security;
 
-import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.SecuritySettingsSourceField;
-import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 
 import java.io.IOException;
 
@@ -31,17 +31,20 @@ protected boolean addMockHttpTransport() {
     public void testThatPluginIsLoaded() throws IOException {
         try {
             logger.info("executing unauthorized request to /_xpack info");
-            getRestClient().performRequest("GET", "/_xpack");
+            getRestClient().performRequest(new Request("GET", "/_xpack"));
             fail("request should have failed");
         } catch(ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(UNAUTHORIZED.getStatus()));
         }
 
         logger.info("executing authorized request to /_xpack infos");
-        Response response = getRestClient().performRequest("GET", "/_xpack",
-                new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                        basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))));
+
+        Request request = new Request("GET", "/_xpack");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader("Authorization", basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+        request.setOptions(options);
+        Response response = getRestClient().performRequest(request);
         assertThat(response.getStatusLine().getStatusCode(), is(OK.getStatus()));
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
index e88b1905a7ab7..857b1694ac87a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
@@ -292,6 +292,26 @@ public void testJoinValidatorForLicenseDeserialization() throws Exception {
         assertThat(e.getMessage(), containsString("cannot deserialize the license format"));
     }
 
+    public void testJoinValidatorForFIPSLicense() throws Exception {
+        DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(),
+            VersionUtils.randomVersionBetween(random(), null, Version.CURRENT));
+        MetaData.Builder builder = MetaData.builder();
+        License license = TestUtils.generateSignedLicense(TimeValue.timeValueHours(24));
+        TestUtils.putLicense(builder, license);
+        ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(builder.build()).build();
+        new Security.ValidateLicenseForFIPS(false).accept(node, state);
+
+        final boolean isLicenseValidForFips =
+            FIPS140LicenseBootstrapCheck.ALLOWED_LICENSE_OPERATION_MODES.contains(license.operationMode());
+        if (isLicenseValidForFips) {
+            new Security.ValidateLicenseForFIPS(true).accept(node, state);
+        } else {
+            IllegalStateException e = expectThrows(IllegalStateException.class,
+                () -> new Security.ValidateLicenseForFIPS(true).accept(node, state));
+            assertThat(e.getMessage(), containsString("FIPS mode cannot be used"));
+        }
+    }
+
     public void testIndexJoinValidator_Old_And_Rolling() throws Exception {
         createComponents(Settings.EMPTY);
         BiConsumer<DiscoveryNode, ClusterState> joinValidator = security.getJoinValidator();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java
index 019901afa28be..1ac5490dc0c6a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java
@@ -33,7 +33,7 @@
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.junit.Before;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java
index 7c951c0014e89..0809276932daa 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 
 import java.util.Collections;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java
index f1363214b0706..f939b175e48b1 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 
 import java.util.Collections;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java
index 158a74308d97f..3371b901647c0 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java
@@ -56,7 +56,7 @@
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.authc.TokenService;
 import org.elasticsearch.xpack.security.authc.UserToken;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java
index 37cbb5ef27940..1ce8b1aff1395 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java
@@ -46,7 +46,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.authc.TokenService;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java
index a8e2464805825..7862097d0006f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 
 import java.util.Collections;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java
index aabaa40381f69..410c164ffe741 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.mockito.invocation.InvocationOnMock;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserActionTests.java
index 4e6e0b3551bc2..0c1ddbd9ba754 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserActionTests.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.mockito.invocation.InvocationOnMock;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java
index 1c5f93187c059..ebdb145559192 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.xpack.core.security.action.user.GetUsersResponse;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java
index a2e283e1b36ff..d7795d3ab9188 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java
@@ -34,7 +34,7 @@
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 import org.hamcrest.Matchers;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java
index fff2479aa5d8b..b7eeb78fad317 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java
index d811b6359b186..4ca7ab97f73d5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.mockito.invocation.InvocationOnMock;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java
index b346fc6857e7e..ba9a67eb48f74 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 import org.junit.Before;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java
index 453820ea519fc..ef3c6aa56aeaf 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java
@@ -5,12 +5,13 @@
  */
 package org.elasticsearch.xpack.security.audit.index;
 
-import org.apache.http.message.BasicHeader;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
@@ -89,10 +90,12 @@ public boolean transportSSLEnabled() {
 
     public void testAuditAccessDeniedWithRunAsUser() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/.security/_search",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(AUTHENTICATE_USER, TEST_PASSWORD_SECURE_STRING)),
-                    new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, EXECUTE_USER));
+            Request request = new Request("GET", "/.security/_search");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(AUTHENTICATE_USER, TEST_PASSWORD_SECURE_STRING));
+            options.addHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, EXECUTE_USER);
+            request.setOptions(options);
+            getRestClient().performRequest(request);
             fail("request should have failed");
         } catch (final ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
@@ -111,10 +114,12 @@ public void testAuditAccessDeniedWithRunAsUser() throws Exception {
 
     public void testAuditRunAsDeniedEmptyUser() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/.security/_search",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(AUTHENTICATE_USER, TEST_PASSWORD_SECURE_STRING)),
-                    new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, ""));
+            Request request = new Request("GET", "/.security/_search");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(AUTHENTICATE_USER, TEST_PASSWORD_SECURE_STRING));
+            options.addHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, "");
+            request.setOptions(options);
+            getRestClient().performRequest(request);
             fail("request should have failed");
         } catch (final ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java
index 9bc5c989d1f9c..923c918f011d8 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail.State;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 import org.junit.After;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java
index cb1b69708bdf2..dcb8d8b7569ce 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java
@@ -55,7 +55,7 @@
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.index.IndexAuditTrailField;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.LocalStateSecurity;
 import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail.Message;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java
index 4c9df8fd9d382..a3a9d05704f0d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java
@@ -28,7 +28,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.AuditEventMetaInfo;
 import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrailTests.MockMessage;
 import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrailTests.RestContent;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
index 1059e22abd663..c8e14e668c911 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
@@ -35,7 +35,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.rest.RemoteHostHeader;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index fda87d0340b8a..4a40e0d543bcf 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -64,7 +64,7 @@
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.authc.AuthenticationService.Authenticator;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
index a71f5cb1cf761..1da7d68c91ca3 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
@@ -23,10 +23,11 @@
 import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -51,13 +52,16 @@ public class RealmsTests extends ESTestCase {
     private XPackLicenseState licenseState;
     private ThreadContext threadContext;
     private ReservedRealm reservedRealm;
+    private int randomRealmTypesCount;
 
     @Before
     public void init() throws Exception {
         factories = new HashMap<>();
         factories.put(FileRealmSettings.TYPE, config -> new DummyRealm(FileRealmSettings.TYPE, config));
         factories.put(NativeRealmSettings.TYPE, config -> new DummyRealm(NativeRealmSettings.TYPE, config));
-        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+        factories.put(KerberosRealmSettings.TYPE, config -> new DummyRealm(KerberosRealmSettings.TYPE, config));
+        randomRealmTypesCount = randomIntBetween(1, 5);
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             String name = "type_" + i;
             factories.put(name, config -> new DummyRealm(name, config));
         }
@@ -73,13 +77,13 @@ public void init() throws Exception {
     public void testWithSettings() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put("path.home", createTempDir());
-        List<Integer> orders = new ArrayList<>(factories.size() - 2);
-        for (int i = 0; i < factories.size() - 2; i++) {
+        List<Integer> orders = new ArrayList<>(randomRealmTypesCount);
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             orders.add(i);
         }
         Collections.shuffle(orders, random());
         Map<Integer, Integer> orderToIndex = new HashMap<>();
-        for (int i = 0; i < factories.size() - 2; i++) {
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             builder.put("xpack.security.authc.realms.realm_" + i + ".type", "type_" + i);
             builder.put("xpack.security.authc.realms.realm_" + i + ".order", orders.get(i));
             orderToIndex.put(orders.get(i), i);
@@ -107,14 +111,14 @@ public void testWithSettings() throws Exception {
     public void testWithSettingsWhereDifferentRealmsHaveSameOrder() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put("path.home", createTempDir());
-        List<Integer> randomSeq = new ArrayList<>(factories.size() - 2);
-        for (int i = 0; i < factories.size() - 2; i++) {
+        List<Integer> randomSeq = new ArrayList<>(randomRealmTypesCount);
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             randomSeq.add(i);
         }
         Collections.shuffle(randomSeq, random());
 
         TreeMap<String, Integer> nameToRealmId = new TreeMap<>();
-        for (int i = 0; i < factories.size() - 2; i++) {
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             int randomizedRealmId = randomSeq.get(i);
             String randomizedRealmName = randomAlphaOfLengthBetween(12,32);
             nameToRealmId.put("realm_" + randomizedRealmName, randomizedRealmId);
@@ -181,13 +185,13 @@ public void testWithEmptySettings() throws Exception {
     public void testUnlicensedWithOnlyCustomRealms() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put("path.home", createTempDir());
-        List<Integer> orders = new ArrayList<>(factories.size() - 2);
-        for (int i = 0; i < factories.size() - 2; i++) {
+        List<Integer> orders = new ArrayList<>(randomRealmTypesCount);
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             orders.add(i);
         }
         Collections.shuffle(orders, random());
         Map<Integer, Integer> orderToIndex = new HashMap<>();
-        for (int i = 0; i < factories.size() - 2; i++) {
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             builder.put("xpack.security.authc.realms.realm_" + i + ".type", "type_" + i);
             builder.put("xpack.security.authc.realms.realm_" + i + ".order", orders.get(i));
             orderToIndex.put(orders.get(i), i);
@@ -384,13 +388,13 @@ public void testUnlicensedWithNonStandardRealms() throws Exception {
     public void testDisabledRealmsAreNotAdded() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put("path.home", createTempDir());
-        List<Integer> orders = new ArrayList<>(factories.size() - 2);
-        for (int i = 0; i < factories.size() - 2; i++) {
+        List<Integer> orders = new ArrayList<>(randomRealmTypesCount);
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             orders.add(i);
         }
         Collections.shuffle(orders, random());
         Map<Integer, Integer> orderToIndex = new HashMap<>();
-        for (int i = 0; i < factories.size() - 2; i++) {
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             builder.put("xpack.security.authc.realms.realm_" + i + ".type", "type_" + i);
             builder.put("xpack.security.authc.realms.realm_" + i + ".order", orders.get(i));
             boolean enabled = randomBoolean();
@@ -520,6 +524,20 @@ public void testUsageStats() throws Exception {
         }
     }
 
+    public void testInitRealmsFailsForMultipleKerberosRealms() throws IOException {
+        final Settings.Builder builder = Settings.builder().put("path.home", createTempDir());
+        builder.put("xpack.security.authc.realms.realm_1.type", "kerberos");
+        builder.put("xpack.security.authc.realms.realm_1.order", 1);
+        builder.put("xpack.security.authc.realms.realm_2.type", "kerberos");
+        builder.put("xpack.security.authc.realms.realm_2.order", 2);
+        final Settings settings = builder.build();
+        Environment env = TestEnvironment.newEnvironment(settings);
+        final IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
+                () -> new Realms(settings, env, factories, licenseState, threadContext, reservedRealm));
+        assertThat(iae.getMessage(), is(equalTo(
+                "multiple realms [realm_1, realm_2] configured of type [kerberos], [kerberos] can only have one such realm configured")));
+    }
+
     static class DummyRealm extends Realm {
 
         DummyRealm(String type, RealmConfig config) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java
index 9311b9c02f60f..6d5c6770bf2f5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.security.authc;
 
-import org.apache.http.message.BasicHeader;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.elasticsearch.client.Response;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.SecureString;
@@ -126,11 +126,13 @@ public void testUserImpersonation() throws Exception {
     public void testUserImpersonationUsingHttp() throws Exception {
         // use the transport client user and try to run as
         try {
-            getRestClient().performRequest("GET", "/_nodes",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(TRANSPORT_CLIENT_USER,
-                                    TEST_PASSWORD_SECURE_STRING)),
-                    new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, SecuritySettingsSource.TEST_USER_NAME));
+            Request request = new Request("GET", "/_nodes");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization",
+                    UsernamePasswordToken.basicAuthHeaderValue(TRANSPORT_CLIENT_USER, TEST_PASSWORD_SECURE_STRING));
+            options.addHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, SecuritySettingsSource.TEST_USER_NAME);
+            request.setOptions(options);
+            getRestClient().performRequest(request);
             fail("request should have failed");
         } catch(ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
@@ -139,10 +141,11 @@ public void testUserImpersonationUsingHttp() throws Exception {
         if (runAsHasSuperUserRole == false) {
             try {
                 //the run as user shouldn't have access to the nodes api
-                getRestClient().performRequest("GET", "/_nodes",
-                        new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                                UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                                        TEST_PASSWORD_SECURE_STRING)));
+                Request request = new Request("GET", "/_nodes");
+                RequestOptions.Builder options = request.getOptions().toBuilder();
+                options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, TEST_PASSWORD_SECURE_STRING));
+                request.setOptions(options);
+                getRestClient().performRequest(request);
                 fail("request should have failed");
             } catch (ResponseException e) {
                 assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
@@ -150,12 +153,7 @@ public void testUserImpersonationUsingHttp() throws Exception {
         }
 
         // but when running as a different user it should work
-        Response response = getRestClient().performRequest("GET", "/_nodes",
-                new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                        UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                                TEST_PASSWORD_SECURE_STRING)),
-                new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, SecuritySettingsSource.TEST_USER_NAME));
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
+        getRestClient().performRequest(requestForUserRunAsUser(SecuritySettingsSource.TEST_USER_NAME));
     }
 
     public void testEmptyUserImpersonationHeader() throws Exception {
@@ -183,11 +181,7 @@ public void testEmptyUserImpersonationHeader() throws Exception {
 
     public void testEmptyHeaderUsingHttp() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/_nodes",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                                    TEST_PASSWORD_SECURE_STRING)),
-                    new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, ""));
+            getRestClient().performRequest(requestForUserRunAsUser(""));
             fail("request should have failed");
         } catch(ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
@@ -219,17 +213,22 @@ public void testNonExistentRunAsUser() throws Exception {
 
     public void testNonExistentRunAsUserUsingHttp() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/_nodes",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                                    TEST_PASSWORD_SECURE_STRING)),
-                    new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, "idontexist"));
+            getRestClient().performRequest(requestForUserRunAsUser("idontexist"));
             fail("request should have failed");
         } catch (ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
         }
     }
 
+    private static Request requestForUserRunAsUser(String user) {
+        Request request = new Request("GET", "/_nodes");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, TEST_PASSWORD_SECURE_STRING));
+        options.addHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, user);
+        request.setOptions(options);
+        return request;
+    }
+
     // build our own here to better mimic an actual client...
     TransportClient getTransportClient(Settings extraSettings) {
         NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
index d5e67f3996a7b..c529ea8747b1e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
@@ -48,7 +48,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.TokenMetaData;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.AfterClass;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/UserTokenTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/UserTokenTests.java
index 1a8f8dc3b5d2d..c79d77718ca44 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/UserTokenTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/UserTokenTests.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import java.io.IOException;
 import java.time.Clock;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
index b7cd23745b932..7f9e25e77e6dd 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
@@ -45,7 +45,7 @@
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
 import org.junit.Before;
 import org.junit.BeforeClass;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java
index d8af70cb6bec7..c7a7c4f07bb6c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.LogstashSystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.Before;
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java
index fad3d43e6d5f3..a03f30abc3366 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.LogstashSystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.UsernamesField;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore.ReservedUserInfo;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java
index f5dad8b7c684c..aec2a4eb8208f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.junit.Before;
 import org.mockito.stubbing.Answer;
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java
index 739952af63ee7..ee89c2efe4f1e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.junit.After;
 import org.junit.Before;
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java
index 5bc239241cf11..11aefb758fb29 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.ietf.jgss.GSSException;
 
 import java.nio.file.Path;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheckTests.java
deleted file mode 100644
index b6e1df9ddbb79..0000000000000
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheckTests.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.authc.kerberos;
-
-import org.elasticsearch.bootstrap.BootstrapCheck;
-import org.elasticsearch.bootstrap.BootstrapContext;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.TestEnvironment;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.security.authc.RealmSettings;
-import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
-import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
-
-import java.io.IOException;
-import java.nio.file.Path;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-
-public class KerberosRealmBootstrapCheckTests extends ESTestCase {
-
-    public void testBootstrapCheckFailsForMultipleKerberosRealms() throws IOException {
-        final Path tempDir = createTempDir();
-        final Settings settings1 = buildKerberosRealmSettings("kerb1", false, tempDir);
-        final Settings settings2 = buildKerberosRealmSettings("kerb2", false, tempDir);
-        final Settings settings3 = realm("pki1", PkiRealmSettings.TYPE, Settings.builder()).build();
-        final Settings settings =
-                Settings.builder().put("path.home", tempDir).put(settings1).put(settings2).put(settings3).build();
-        final BootstrapContext context = new BootstrapContext(settings, null);
-        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
-                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(settings));
-        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
-        assertThat(result, is(notNullValue()));
-        assertThat(result.isFailure(), is(true));
-        assertThat(result.getMessage(), equalTo("multiple [" + KerberosRealmSettings.TYPE + "] realms are configured. ["
-                + KerberosRealmSettings.TYPE + "] can only have one such realm configured"));
-    }
-
-    public void testBootstrapCheckFailsForMissingKeytabFile() throws IOException {
-        final Path tempDir = createTempDir();
-        final Settings settings =
-                Settings.builder().put("path.home", tempDir).put(buildKerberosRealmSettings("kerb1", true, tempDir)).build();
-        final BootstrapContext context = new BootstrapContext(settings, null);
-        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
-                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(settings));
-        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
-        assertThat(result, is(notNullValue()));
-        assertThat(result.isFailure(), is(true));
-        assertThat(result.getMessage(),
-                equalTo("configured service key tab file [" + tempDir.resolve("kerb1.keytab").toString() + "] does not exist"));
-    }
-
-    public void testBootstrapCheckFailsForMissingRealmType() throws IOException {
-        final Path tempDir = createTempDir();
-        final String name = "kerb1";
-        final Settings settings1 = buildKerberosRealmSettings("kerb1", false, tempDir);
-        final Settings settings2 = realm(name, randomFrom("", "    "), Settings.builder()).build();
-        final Settings settings =
-                Settings.builder().put("path.home", tempDir).put(settings1).put(settings2).build();
-        final BootstrapContext context = new BootstrapContext(settings, null);
-        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
-                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(settings));
-        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
-        assertThat(result, is(notNullValue()));
-        assertThat(result.isFailure(), is(true));
-        assertThat(result.getMessage(), equalTo("missing realm type for [" + name + "] realm"));
-    }
-
-    public void testBootstrapCheckSucceedsForCorrectConfiguration() throws IOException {
-        final Path tempDir = createTempDir();
-        final Settings finalSettings =
-                Settings.builder().put("path.home", tempDir).put(buildKerberosRealmSettings("kerb1", false, tempDir)).build();
-        final BootstrapContext context = new BootstrapContext(finalSettings, null);
-        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
-                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(finalSettings));
-        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
-        assertThat(result, is(notNullValue()));
-        assertThat(result.isSuccess(), is(true));
-    }
-
-    public void testBootstrapCheckSucceedsForNoKerberosRealms() throws IOException {
-        final Path tempDir = createTempDir();
-        final Settings finalSettings = Settings.builder().put("path.home", tempDir).build();
-        final BootstrapContext context = new BootstrapContext(finalSettings, null);
-        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
-                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(finalSettings));
-        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
-        assertThat(result, is(notNullValue()));
-        assertThat(result.isSuccess(), is(true));
-    }
-
-    private Settings buildKerberosRealmSettings(final String name, final boolean missingKeytab, final Path tempDir) throws IOException {
-        final Settings.Builder builder = Settings.builder();
-        if (missingKeytab == false) {
-            KerberosTestCase.writeKeyTab(tempDir.resolve(name + ".keytab"), null);
-        }
-        builder.put(KerberosTestCase.buildKerberosRealmSettings(tempDir.resolve(name + ".keytab").toString()));
-        return realm(name, KerberosRealmSettings.TYPE, builder).build();
-    }
-
-    private Settings.Builder realm(final String name, final String type, final Settings.Builder settings) {
-        final String prefix = RealmSettings.PREFIX + name + ".";
-        if (type != null) {
-            settings.put("type", type);
-        }
-        final Settings.Builder builder = Settings.builder().put(settings.normalizePrefix(prefix).build(), false);
-        return builder;
-    }
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java
index 69ebe15c5d74b..7ce8ee39c038d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.support.UserRoleMapper.UserData;
 import org.ietf.jgss.GSSException;
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java
index 9c2c6484c82ab..69b246cd7ca6c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java
index 43536abaf29e1..80e61a5545fea 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java
@@ -11,17 +11,32 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.support.UserRoleMapper.UserData;
 import org.ietf.jgss.GSSException;
 
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.AclEntry;
+import java.nio.file.attribute.AclEntryPermission;
+import java.nio.file.attribute.AclEntryType;
+import java.nio.file.attribute.AclFileAttributeView;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.nio.file.attribute.UserPrincipal;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
 
 import javax.security.auth.login.LoginException;
 
@@ -31,6 +46,7 @@
 import static org.mockito.AdditionalMatchers.aryEq;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -94,4 +110,59 @@ public void testLookupUser() {
         assertThat(future.actionGet(), is(nullValue()));
     }
 
-}
\ No newline at end of file
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32768")
+    public void testKerberosRealmWithInvalidKeytabPathConfigurations() throws IOException {
+        final String keytabPathCase = randomFrom("keytabPathAsDirectory", "keytabFileDoesNotExist", "keytabPathWithNoReadPermissions");
+        final String expectedErrorMessage;
+        final String keytabPath;
+        switch (keytabPathCase) {
+        case "keytabPathAsDirectory":
+            final String dirName = randomAlphaOfLength(5);
+            Files.createDirectory(dir.resolve(dirName));
+            keytabPath = dir.resolve(dirName).toString();
+            expectedErrorMessage = "configured service key tab file [" + keytabPath + "] is a directory";
+            break;
+        case "keytabFileDoesNotExist":
+            keytabPath = dir.resolve(randomAlphaOfLength(5) + ".keytab").toString();
+            expectedErrorMessage = "configured service key tab file [" + keytabPath + "] does not exist";
+            break;
+        case "keytabPathWithNoReadPermissions":
+            final String fileName = randomAlphaOfLength(5);
+            final Path keytabFilePath = Files.createTempFile(dir, fileName, ".keytab");
+            Files.write(keytabFilePath, randomAlphaOfLength(5).getBytes(StandardCharsets.UTF_8));
+            final Set<String> supportedAttributes = keytabFilePath.getFileSystem().supportedFileAttributeViews();
+            if (supportedAttributes.contains("posix")) {
+                final PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(keytabFilePath, PosixFileAttributeView.class);
+                fileAttributeView.setPermissions(PosixFilePermissions.fromString("---------"));
+            } else if (supportedAttributes.contains("acl")) {
+                final UserPrincipal principal = Files.getOwner(keytabFilePath);
+                final AclFileAttributeView view = Files.getFileAttributeView(keytabFilePath, AclFileAttributeView.class);
+                final AclEntry entry = AclEntry.newBuilder()
+                        .setType(AclEntryType.DENY)
+                        .setPrincipal(principal)
+                        .setPermissions(AclEntryPermission.READ_DATA, AclEntryPermission.READ_ATTRIBUTES).build();
+                final List<AclEntry> acl = view.getAcl();
+                acl.add(0, entry);
+                view.setAcl(acl);
+            } else {
+                throw new UnsupportedOperationException(
+                        String.format(Locale.ROOT, "Don't know how to make file [%s] non-readable on a file system with attributes [%s]",
+                                keytabFilePath, supportedAttributes));
+            }
+            keytabPath = keytabFilePath.toString();
+            expectedErrorMessage = "configured service key tab file [" + keytabPath + "] must have read permission";
+            break;
+        default:
+            throw new IllegalArgumentException("Unknown test case :" + keytabPathCase);
+        }
+
+        settings = KerberosTestCase.buildKerberosRealmSettings(keytabPath, 100, "10m", true, randomBoolean());
+        config = new RealmConfig("test-kerb-realm", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
+                new ThreadContext(globalSettings));
+        mockNativeRoleMappingStore = roleMappingStore(Arrays.asList("user"));
+        mockKerberosTicketValidator = mock(KerberosTicketValidator.class);
+        final IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
+                () -> new KerberosRealm(config, mockNativeRoleMappingStore, mockKerberosTicketValidator, threadPool, null));
+        assertThat(iae.getMessage(), is(equalTo(expectedErrorMessage)));
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
index 891f400c7be60..0f88148a9a973 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
@@ -70,6 +70,8 @@ public abstract class KerberosTestCase extends ESTestCase {
         unsupportedLocaleLanguages.add("uz");
         unsupportedLocaleLanguages.add("fa");
         unsupportedLocaleLanguages.add("ks");
+        unsupportedLocaleLanguages.add("ckb");
+        unsupportedLocaleLanguages.add("ne");
     }
 
     @BeforeClass
@@ -133,7 +135,7 @@ public void tearDownMiniKdc() throws IOException, PrivilegedActionException {
      * @param dir Directory where the key tab would be created.
      * @param princNames principal names to be created
      * @return {@link Path} to key tab file.
-     * @throws Exception
+     * @throws Exception thrown if principal or keytab could not be created
      */
     protected Path createPrincipalKeyTab(final Path dir, final String... princNames) throws Exception {
         final Path path = dir.resolve(randomAlphaOfLength(10) + ".keytab");
@@ -143,10 +145,10 @@ protected Path createPrincipalKeyTab(final Path dir, final String... princNames)
 
     /**
      * Creates principal with given name and password.
-     * 
+     *
      * @param principalName Principal name
      * @param password Password
-     * @throws Exception
+     * @throws Exception thrown if principal could not be created
      */
     protected void createPrincipal(final String principalName, final char[] password) throws Exception {
         simpleKdcLdapServer.createPrincipal(principalName, new String(password));
@@ -168,8 +170,8 @@ protected String principalName(final String user) {
      * @param subject {@link Subject}
      * @param action {@link PrivilegedExceptionAction} action for performing inside
      *            Subject.doAs
-     * @return <T> Type of value as returned by PrivilegedAction
-     * @throws PrivilegedActionException
+     * @return T Type of value as returned by PrivilegedAction
+     * @throws PrivilegedActionException when privileged action threw exception
      */
     static <T> T doAsWrapper(final Subject subject, final PrivilegedExceptionAction<T> action) throws PrivilegedActionException {
         return AccessController.doPrivileged((PrivilegedExceptionAction<T>) () -> Subject.doAs(subject, action));
@@ -177,11 +179,11 @@ static <T> T doAsWrapper(final Subject subject, final PrivilegedExceptionAction<
 
     /**
      * Write content to provided keytab file.
-     * 
+     *
      * @param keytabPath {@link Path} to keytab file.
      * @param content Content for keytab
      * @return key tab path
-     * @throws IOException
+     * @throws IOException if I/O error occurs while writing keytab file
      */
     public static Path writeKeyTab(final Path keytabPath, final String content) throws IOException {
         try (BufferedWriter bufferedWriter = Files.newBufferedWriter(keytabPath, StandardCharsets.US_ASCII)) {
@@ -192,7 +194,7 @@ public static Path writeKeyTab(final Path keytabPath, final String content) thro
 
     /**
      * Build kerberos realm settings with default config and given keytab
-     * 
+     *
      * @param keytabPath key tab file path
      * @return {@link Settings} for kerberos realm
      */
@@ -202,7 +204,7 @@ public static Settings buildKerberosRealmSettings(final String keytabPath) {
 
     /**
      * Build kerberos realm settings
-     * 
+     *
      * @param keytabPath key tab file path
      * @param maxUsersInCache max users to be maintained in cache
      * @param cacheTTL time to live for cached entries
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
index 426cacb1a034c..53bf9e2b78d0b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
@@ -66,7 +66,7 @@ public class SimpleKdcLdapServer {
      * @param orgName Org name for base dn
      * @param domainName domain name for base dn
      * @param ldiff for ldap directory.
-     * @throws Exception
+     * @throws Exception when KDC or Ldap server initialization fails
      */
     public SimpleKdcLdapServer(final Path workDir, final String orgName, final String domainName, final Path ldiff) throws Exception {
         this.workDir = workDir;
@@ -194,7 +194,7 @@ public synchronized void createPrincipal(final Path keytabFile, final String...
     /**
      * Stop Simple Kdc Server
      * 
-     * @throws PrivilegedActionException
+     * @throws PrivilegedActionException when privileged action threw exception
      */
     public synchronized void stop() throws PrivilegedActionException {
         AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java
index 1f883b928bd97..7fcca3d9f5806 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.security.authc.kerberos.KerberosTicketValidator;
 import org.ietf.jgss.GSSContext;
 import org.ietf.jgss.GSSCredential;
@@ -67,8 +66,8 @@ class SpnegoClient implements AutoCloseable {
      * @param password password for client
      * @param servicePrincipalName Service principal name with whom this client
      *            interacts with.
-     * @throws PrivilegedActionException
-     * @throws GSSException
+     * @throws PrivilegedActionException when privileged action threw exception
+     * @throws GSSException thrown when GSS API error occurs
      */
     SpnegoClient(final String userPrincipalName, final SecureString password, final String servicePrincipalName)
             throws PrivilegedActionException, GSSException {
@@ -99,7 +98,7 @@ class SpnegoClient implements AutoCloseable {
      * base64 encoded token to be sent to server.
      *
      * @return Base64 encoded token
-     * @throws PrivilegedActionException
+     * @throws PrivilegedActionException when privileged action threw exception
      */
     String getBase64EncodedTokenForSpnegoHeader() throws PrivilegedActionException {
         final byte[] outToken = KerberosTestCase.doAsWrapper(loginContext.getSubject(),
@@ -114,7 +113,7 @@ String getBase64EncodedTokenForSpnegoHeader() throws PrivilegedActionException {
      *            gss negotiation
      * @return Base64 encoded token to be sent to server. May return {@code null} if
      *         nothing to be sent.
-     * @throws PrivilegedActionException
+     * @throws PrivilegedActionException when privileged action threw exception
      */
     String handleResponse(final String base64Token) throws PrivilegedActionException {
         if (gssContext.isEstablished()) {
@@ -160,10 +159,9 @@ boolean isEstablished() {
      *
      * @param principal Principal name
      * @param password {@link SecureString}
-     * @param settings {@link Settings}
      * @return authenticated {@link LoginContext} instance. Note: This needs to be
      *         closed {@link LoginContext#logout()} after usage.
-     * @throws LoginException
+     * @throws LoginException thrown if problem with login configuration or when login fails
      */
     private static LoginContext loginUsingPassword(final String principal, final SecureString password) throws LoginException {
         final Set<Principal> principals = Collections.singleton(new KerberosPrincipal(principal));
@@ -182,8 +180,8 @@ private static LoginContext loginUsingPassword(final String principal, final Sec
      * Instead of an additional file setting as we do not want the options to be
      * customizable we are constructing it in memory.
      * <p>
-     * As we are uing this instead of jaas.conf, this requires refresh of
-     * {@link Configuration} and reqires appropriate security permissions to do so.
+     * As we are using this instead of jaas.conf, this requires refresh of
+     * {@link Configuration} and requires appropriate security permissions to do so.
      */
     static class PasswordJaasConf extends Configuration {
         private final String principal;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java
index 2c6756aada7ac..a8f555bc3a392 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java
@@ -35,7 +35,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.DnRoleMapperSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.ssl.VerificationMode;
 import org.elasticsearch.xpack.security.authc.ldap.ActiveDirectorySessionFactory.DownLevelADAuthenticator;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/CancellableLdapRunnableTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/CancellableLdapRunnableTests.java
index 18b84df6d6185..2807c501a5d6d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/CancellableLdapRunnableTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/CancellableLdapRunnableTests.java
@@ -8,7 +8,7 @@
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.ldap.LdapRealm.CancellableLdapRunnable;
 
 import java.util.concurrent.CountDownLatch;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java
index 4aff821217d14..5c98e2347cf12 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.DnRoleMapperSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.ssl.VerificationMode;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapTestCase;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryTests.java
index d82eb520e9f18..6540be6a5eb14 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryTests.java
@@ -93,10 +93,13 @@ public void testConnectionFactoryReturnsCorrectLDAPConnectionOptions() throws Ex
         options = SessionFactory.connectionOptions(realmConfig, sslService.apply(settings), logger);
         assertThat(options.getSSLSocketVerifier(), is(instanceOf(TrustAllSSLSocketVerifier.class)));
 
-        settings = Settings.builder().put("ssl.verification_mode", VerificationMode.NONE).build();
-        realmConfig = new RealmConfig(realmName, settings, environment.settings(), environment, threadContext);
-        options = SessionFactory.connectionOptions(realmConfig, sslService.apply(settings), logger);
-        assertThat(options.getSSLSocketVerifier(), is(instanceOf(TrustAllSSLSocketVerifier.class)));
+        // Can't run in FIPS with verification_mode none, disable this check instead of duplicating the test case
+        if (inFipsJvm() == false) {
+            settings = Settings.builder().put("ssl.verification_mode", VerificationMode.NONE).build();
+            realmConfig = new RealmConfig(realmName, settings, environment.settings(), environment, threadContext);
+            options = SessionFactory.connectionOptions(realmConfig, sslService.apply(settings), logger);
+            assertThat(options.getSSLSocketVerifier(), is(instanceOf(TrustAllSSLSocketVerifier.class)));
+        }
 
         settings = Settings.builder().put("ssl.verification_mode", VerificationMode.FULL).build();
         realmConfig = new RealmConfig(realmName, settings, environment.settings(), environment, threadContext);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java
index d47ffe4a3446c..4fb94c7494971 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java
@@ -5,8 +5,9 @@
  */
 package org.elasticsearch.xpack.security.authc.pki;
 
-import org.apache.http.message.BasicHeader;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -76,13 +77,15 @@ protected boolean transportSSLEnabled() {
     public void testRestClientWithoutClientCertificate() throws Exception {
         SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(getSSLContext());
         try (RestClient restClient = createRestClient(httpClientBuilder -> httpClientBuilder.setSSLStrategy(sessionStrategy), "https")) {
-            ResponseException e = expectThrows(ResponseException.class, () -> restClient.performRequest("GET", "_nodes"));
+            ResponseException e = expectThrows(ResponseException.class, () -> restClient.performRequest(new Request("GET", "_nodes")));
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
 
-            Response response = restClient.performRequest("GET", "_nodes",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                                    new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))));
+            Request request = new Request("GET", "_nodes");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                    new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+            request.setOptions(options);
+            Response response = restClient.performRequest(request);
             assertThat(response.getStatusLine().getStatusCode(), is(200));
         }
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java
index 44d5859d12b67..2410d8c464997 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.support.NoOpLogger;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
 import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
 import org.junit.Before;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java
index 052758d83718c..e6830be18c58a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.junit.After;
 import org.junit.Before;
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
index 052ba38551021..2bee8fa09e381 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression.FieldValue;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm;
 import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
index 7722a9d216632..65c558d3d81e2 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
@@ -122,7 +122,7 @@
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java
index 9c9f2b1b1a42a..a581d1abbb5df 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackSecurityUser;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.junit.Before;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java
index c48ac4568989b..d31f9f37b9158 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
index eed3297661a47..cf9c09759ea09 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
@@ -60,7 +60,7 @@
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackSecurityUser;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index fac88e8af09b0..087749da2406a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 
 import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java
index 26c59a1ef5470..05c2882f3ac12 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor.Property;
 
 import java.util.Collections;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java
index 67bfc2ecdcb0d..13a124e4bdc96 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java
@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.xpack.security.rest.action;
 
-import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.SecureString;
@@ -52,11 +53,12 @@ protected Settings nodeSettings(int nodeOrdinal) {
     }
 
     public void testAuthenticateApi() throws Exception {
-        Response response = getRestClient().performRequest("GET", "/_xpack/security/_authenticate",
-                new BasicHeader("Authorization", basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                        new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))));
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        Request request = new Request("GET", "/_xpack/security/_authenticate");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader("Authorization", basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+        request.setOptions(options);
+        ObjectPath objectPath = ObjectPath.createFromResponse(getRestClient().performRequest(request));
         assertThat(objectPath.evaluate("username").toString(), equalTo(SecuritySettingsSource.TEST_USER_NAME));
         List<String> roles = objectPath.evaluate("roles");
         assertThat(roles.size(), is(1));
@@ -65,7 +67,7 @@ public void testAuthenticateApi() throws Exception {
 
     public void testAuthenticateApiWithoutAuthentication() throws Exception {
         try {
-            Response response = getRestClient().performRequest("GET", "/_xpack/security/_authenticate");
+            Response response = getRestClient().performRequest(new Request("GET", "/_xpack/security/_authenticate"));
             if (anonymousEnabled) {
                 assertThat(response.getStatusLine().getStatusCode(), is(200));
                 ObjectPath objectPath = ObjectPath.createFromResponse(response);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
index dd7dda48ae813..09072f99fc20b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
@@ -33,7 +33,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java
index 38b9a029d9bd1..08a991eb3ec29 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransportTests.java
index b5d84d459160f..9c490176eccdb 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransportTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransportTests.java
@@ -49,11 +49,13 @@ public class SecurityNioHttpServerTransportTests extends ESTestCase {
 
     @Before
     public void createSSLService() {
-        Path testNodeStore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks");
+        Path testNodeKey = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem");
+        Path testNodeCert = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-            .put("xpack.ssl.keystore.path", testNodeStore)
+            .put("xpack.ssl.key", testNodeKey)
+            .put("xpack.ssl.certificate", testNodeCert)
             .put("path.home", createTempDir())
             .setSecureSettings(secureSettings)
             .build();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java
index 9529a12a30ab8..431b3e855c68d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.security.user;
 
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
@@ -46,7 +47,7 @@ public String configRoles() {
 
     public void testAnonymousViaHttp() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/_nodes");
+            getRestClient().performRequest(new Request("GET", "/_nodes"));
             fail("request should have failed");
         } catch(ResponseException e) {
             int statusCode = e.getResponse().getStatusLine().getStatusCode();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserTests.java
index 4c72afeb5cee7..32816e40e0873 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserTests.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserSerializationTests.java
similarity index 89%
rename from x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserTests.java
rename to x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserSerializationTests.java
index 5be4b1c0eca2e..6bea620982fac 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserSerializationTests.java
@@ -13,18 +13,17 @@
 import org.elasticsearch.xpack.core.security.user.InternalUserSerializationHelper;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 
 import java.util.Arrays;
-import java.util.Collections;
 
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
-public class UserTests extends ESTestCase {
+public class UserSerializationTests extends ESTestCase {
 
     public void testWriteToAndReadFrom() throws Exception {
         User user = new User(randomAlphaOfLengthBetween(4, 30),
@@ -142,16 +141,6 @@ public void testFakeInternalUserSerialization() throws Exception {
         }
     }
 
-    public void testUserToString() throws Exception {
-        User user = new User("u1", "r1");
-        assertThat(user.toString(), is("User[username=u1,roles=[r1],fullName=null,email=null,metadata={}]"));
-        user = new User("u1", new String[] { "r1", "r2" }, "user1", "user1@domain.com", Collections.singletonMap("key", "val"), true);
-        assertThat(user.toString(), is("User[username=u1,roles=[r1,r2],fullName=user1,email=user1@domain.com,metadata={key=val}]"));
-        user = new User("u1", new String[] {"r1"}, new User("u2", "r2", "r3"));
-        assertThat(user.toString(), is("User[username=u1,roles=[r1],fullName=null,email=null,metadata={}," +
-                "authenticatedUser=[User[username=u2,roles=[r2,r3],fullName=null,email=null,metadata={}]]]"));
-    }
-
     public void testReservedUserSerialization() throws Exception {
         BytesStreamOutput output = new BytesStreamOutput();
         final ElasticUser elasticUser = new ElasticUser(true);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
index d205c7cd9334e..21da604374f7a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
@@ -6,12 +6,13 @@
 package org.elasticsearch.xpack.ssl;
 
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.message.BasicHeader;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.transport.TransportClient;
@@ -71,7 +72,7 @@ protected boolean transportSSLEnabled() {
     public void testThatHttpFailsWithoutSslClientAuth() throws IOException {
         SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(SSLContexts.createDefault(), NoopHostnameVerifier.INSTANCE);
         try (RestClient restClient = createRestClient(httpClientBuilder -> httpClientBuilder.setSSLStrategy(sessionStrategy), "https")) {
-            restClient.performRequest("GET", "/");
+            restClient.performRequest(new Request("GET", "/"));
             fail("Expected SSLHandshakeException");
         } catch (IOException e) {
             Throwable t = ExceptionsHelper.unwrap(e, CertPathBuilderException.class);
@@ -87,8 +88,11 @@ public void testThatHttpFailsWithoutSslClientAuth() throws IOException {
     public void testThatHttpWorksWithSslClientAuth() throws IOException {
         SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(getSSLContext(), NoopHostnameVerifier.INSTANCE);
         try (RestClient restClient = createRestClient(httpClientBuilder -> httpClientBuilder.setSSLStrategy(sessionStrategy), "https")) {
-            Response response = restClient.performRequest("GET", "/",
-                    new BasicHeader("Authorization", basicAuthHeaderValue(transportClientUsername(), transportClientPassword())));
+            Request request = new Request("GET", "/");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", basicAuthHeaderValue(transportClientUsername(), transportClientPassword()));
+            request.setOptions(options);
+            Response response = restClient.performRequest(request);
             assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
             assertThat(EntityUtils.toString(response.getEntity()), containsString("You Know, for Search"));
         }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
index cf77ca975a4df..e1896e01365a8 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
@@ -7,6 +7,7 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -173,6 +174,8 @@ public void testCertificateWithTrustedNameIsAccepted() throws Exception {
     }
 
     public void testCertificateWithUntrustedNameFails() throws Exception {
+        // see https://github.com/elastic/elasticsearch/issues/29989
+        assumeTrue("test fails on JDK 11 currently", JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0);
         writeRestrictions("*.trusted");
         try {
             tryConnect(untrustedCert);
@@ -183,6 +186,8 @@ public void testCertificateWithUntrustedNameFails() throws Exception {
     }
 
     public void testRestrictionsAreReloaded() throws Exception {
+        // see https://github.com/elastic/elasticsearch/issues/29989
+        assumeTrue("test fails on JDK 11 currently", JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0);
         writeRestrictions("*");
         assertBusy(() -> {
             try {
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.10.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.10.jar.sha1
deleted file mode 100644
index a322d371e265e..0000000000000
--- a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-eb21a035c66ad307e66ec8fce37f5d50fd62d039
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/build.gradle b/x-pack/plugin/sql/sql-action/build.gradle
index f6b5177d50833..bf79fd824ef8d 100644
--- a/x-pack/plugin/sql/sql-action/build.gradle
+++ b/x-pack/plugin/sql/sql-action/build.gradle
@@ -76,8 +76,6 @@ thirdPartyAudit.excludes = [
         'com.fasterxml.jackson.dataformat.xml.JacksonXmlModule',
         'com.fasterxml.jackson.dataformat.xml.XmlMapper',
         'com.fasterxml.jackson.dataformat.xml.util.DefaultXmlPrettyPrinter',
-        'com.fasterxml.jackson.databind.node.JsonNodeFactory',
-        'com.fasterxml.jackson.databind.node.ObjectNode',
         'com.lmax.disruptor.BlockingWaitStrategy',
         'com.lmax.disruptor.BusySpinWaitStrategy',
         'com.lmax.disruptor.EventFactory',
@@ -116,12 +114,6 @@ thirdPartyAudit.excludes = [
         'javax.mail.internet.MimeMultipart',
         'javax.mail.internet.MimeUtility',
         'javax.mail.util.ByteArrayDataSource',
-        'javax.persistence.AttributeConverter',
-        'javax.persistence.EntityManager',
-        'javax.persistence.EntityManagerFactory',
-        'javax.persistence.EntityTransaction',
-        'javax.persistence.Persistence',
-        'javax.persistence.PersistenceException',
         'org.apache.commons.compress.compressors.CompressorStreamFactory',
         'org.apache.commons.compress.utils.IOUtils',
         'org.apache.commons.csv.CSVFormat',
@@ -150,3 +142,13 @@ thirdPartyAudit.excludes = [
         'org.zeromq.ZMQ$Socket',
         'org.zeromq.ZMQ'
 ]
+
+if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+    // Used by Log4J 2.11.1
+    thirdPartyAudit.excludes += [
+      'java.io.ObjectInputFilter',
+      'java.io.ObjectInputFilter$Config',
+      'java.io.ObjectInputFilter$FilterInfo',
+      'java.io.ObjectInputFilter$Status'
+    ]
+}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.10.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.10.jar.sha1
deleted file mode 100644
index a322d371e265e..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-eb21a035c66ad307e66ec8fce37f5d50fd62d039
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/log4j-api-2.11.1.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/log4j-api-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..4b1bfffac179f
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/log4j-api-2.11.1.jar.sha1
@@ -0,0 +1 @@
+268f0fe4df3eefe052b57c87ec48517d64fb2a10
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/log4j-api-2.9.1.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/log4j-api-2.9.1.jar.sha1
deleted file mode 100644
index e1a89fadfed95..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/log4j-api-2.9.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7a2999229464e7a324aa503c0a52ec0f05efe7bd
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/log4j-core-2.11.1.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/log4j-core-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..2fb8589380a03
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/log4j-core-2.11.1.jar.sha1
@@ -0,0 +1 @@
+592a48674c926b01a9a747c7831bcd82a9e6d6e4
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/log4j-core-2.9.1.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/log4j-core-2.9.1.jar.sha1
deleted file mode 100644
index 990ea322a7613..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/log4j-core-2.9.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c041978c686866ee8534f538c6220238db3bb6be
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-13b9e28f9d.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-13b9e28f9d.jar.sha1
new file mode 100644
index 0000000000000..fdedaf3fc5756
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-13b9e28f9d.jar.sha1
@@ -0,0 +1 @@
+73dd7703a94ec2357581f65ee7c1c4d618ff310f
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1
deleted file mode 100644
index d38fb392c350b..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-471096d6e92338b208aa91f3a85feb2f9cfc4afd
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.10.jar.sha1 b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.10.jar.sha1
deleted file mode 100644
index a322d371e265e..0000000000000
--- a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-eb21a035c66ad307e66ec8fce37f5d50fd62d039
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.10.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.10.jar.sha1
deleted file mode 100644
index a322d371e265e..0000000000000
--- a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-eb21a035c66ad307e66ec8fce37f5d50fd62d039
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java
index 212149683ffd9..c9d652861f800 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java
@@ -68,6 +68,15 @@
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.RTrim;
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.Space;
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.UCase;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.Concat;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.Insert;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.Left;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.Locate;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.Position;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.Repeat;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.Replace;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.Right;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.Substring;
 import org.elasticsearch.xpack.sql.parser.ParsingException;
 import org.elasticsearch.xpack.sql.tree.Location;
 import org.elasticsearch.xpack.sql.util.StringUtils;
@@ -154,6 +163,15 @@ public class FunctionRegistry {
             def(LTrim.class, LTrim::new),
             def(RTrim.class, RTrim::new),
             def(Space.class, Space::new),
+            def(Concat.class, Concat::new),
+            def(Insert.class, Insert::new),
+            def(Left.class, Left::new),
+            def(Locate.class, Locate::new),
+            def(Position.class, Position::new),
+            def(Repeat.class, Repeat::new),
+            def(Replace.class, Replace::new),
+            def(Right.class, Right::new),
+            def(Substring.class, Substring::new),
             def(UCase.class, UCase::new),
         // Special
             def(Score.class, Score::new)));
@@ -337,6 +355,47 @@ private static FunctionDefinition def(Class<? extends Function> function, Functi
     private interface FunctionBuilder {
         Function build(Location location, List<Expression> children, boolean distinct, TimeZone tz);
     }
+    
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    static <T extends Function> FunctionDefinition def(Class<T> function,
+            ThreeParametersFunctionBuilder<T> ctorRef, String... aliases) {
+        FunctionBuilder builder = (location, children, distinct, tz) -> {
+            boolean isLocateFunction = function.isAssignableFrom(Locate.class);
+            if (isLocateFunction && (children.size() > 3 || children.size() < 2)) {
+                throw new IllegalArgumentException("expects two or three arguments");
+            } else if (!isLocateFunction && children.size() != 3) {
+                throw new IllegalArgumentException("expects exactly three arguments");
+            }
+            if (distinct) {
+                throw new IllegalArgumentException("does not support DISTINCT yet it was specified");
+            }
+            return ctorRef.build(location, children.get(0), children.get(1), children.size() == 3 ? children.get(2) : null);
+        };
+        return def(function, builder, false, aliases);
+    }
+    
+    interface ThreeParametersFunctionBuilder<T> {
+        T build(Location location, Expression source, Expression exp1, Expression exp2);
+    }
+    
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    static <T extends Function> FunctionDefinition def(Class<T> function,
+            FourParametersFunctionBuilder<T> ctorRef, String... aliases) {
+        FunctionBuilder builder = (location, children, distinct, tz) -> {
+            if (children.size() != 4) {
+                throw new IllegalArgumentException("expects exactly four arguments");
+            }
+            if (distinct) {
+                throw new IllegalArgumentException("does not support DISTINCT yet it was specified");
+            }
+            return ctorRef.build(location, children.get(0), children.get(1), children.get(2), children.get(3));
+        };
+        return def(function, builder, false, aliases);
+    }
+    
+    interface FourParametersFunctionBuilder<T> {
+        T build(Location location, Expression source, Expression exp1, Expression exp2, Expression exp3);
+    }
 
     private static String normalize(String name) {
         // translate CamelCase to camel_case
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java
index 3c3f629cc1caf..0f36654fa4aff 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java
@@ -1,5 +1,5 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
@@ -18,6 +18,13 @@
 import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.HitExtractorProcessor;
 import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor;
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringNumericProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringStringProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.ConcatFunctionProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.InsertFunctionProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.LocateFunctionProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.ReplaceFunctionProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.SubstringFunctionProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -49,6 +56,13 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(new Entry(Processor.class, MathProcessor.NAME, MathProcessor::new));
         // string
         entries.add(new Entry(Processor.class, StringProcessor.NAME, StringProcessor::new));
+        entries.add(new Entry(Processor.class, BinaryStringNumericProcessor.NAME, BinaryStringNumericProcessor::new));
+        entries.add(new Entry(Processor.class, BinaryStringStringProcessor.NAME, BinaryStringStringProcessor::new));
+        entries.add(new Entry(Processor.class, ConcatFunctionProcessor.NAME, ConcatFunctionProcessor::new));
+        entries.add(new Entry(Processor.class, InsertFunctionProcessor.NAME, InsertFunctionProcessor::new));
+        entries.add(new Entry(Processor.class, LocateFunctionProcessor.NAME, LocateFunctionProcessor::new));
+        entries.add(new Entry(Processor.class, ReplaceFunctionProcessor.NAME, ReplaceFunctionProcessor::new));
+        entries.add(new Entry(Processor.class, SubstringFunctionProcessor.NAME, SubstringFunctionProcessor::new));
         return entries;
     }
 }
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringFunction.java
new file mode 100644
index 0000000000000..2ee241c8a0fe7
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringFunction.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.FieldAttribute;
+import org.elasticsearch.xpack.sql.expression.function.scalar.BinaryScalarFunction;
+import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.type.DataType;
+import org.elasticsearch.xpack.sql.util.StringUtils;
+
+import java.util.Locale;
+import java.util.Objects;
+import java.util.function.BiFunction;
+
+import static java.lang.String.format;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate;
+
+/**
+ * Base class for binary functions that have the first parameter a string, the second parameter a number
+ * or a string and the result can be a string or a number.
+ */
+public abstract class BinaryStringFunction<T,R> extends BinaryScalarFunction {
+
+    protected BinaryStringFunction(Location location, Expression left, Expression right) {
+        super(location, left, right);
+    }
+
+    /*
+     * the operation the binary function handles can receive one String argument, a number or String as second argument
+     * and it can return a number or a String. The BiFunction below is the base operation for the subsequent implementations.
+     * T is the second argument, R is the result of applying the operation.
+     */
+    protected abstract BiFunction<String, T, R> operation();
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        if (!left().dataType().isString()) {
+            return new TypeResolution("'%s' requires first parameter to be a string type, received %s", functionName(), left().dataType());
+        }
+                
+        return resolveSecondParameterInputType(right().dataType());
+    }
+
+    protected abstract TypeResolution resolveSecondParameterInputType(DataType inputType);
+
+    @Override
+    public Object fold() {
+        @SuppressWarnings("unchecked")
+        T fold = (T) right().fold();
+        return operation().apply((String) left().fold(), fold);
+    }
+
+    @Override
+    protected ScriptTemplate asScriptFrom(ScriptTemplate leftScript, ScriptTemplate rightScript) {
+        // basically, transform the script to InternalSqlScriptUtils.[function_name](function_or_field1, function_or_field2)
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{sql}.%s(%s,%s)"), 
+                StringUtils.underscoreToLowerCamelCase(operation().toString()), 
+                leftScript.template(), 
+                rightScript.template()),
+                paramsBuilder()
+                    .script(leftScript.params()).script(rightScript.params())
+                    .build(), dataType());
+    }
+    
+    @Override
+    protected ScriptTemplate asScriptFrom(FieldAttribute field) {
+        return new ScriptTemplate(formatScript("doc[{}].value"),
+                paramsBuilder().variable(field.isInexact() ? field.exactAttribute().name() : field.name()).build(),
+                dataType());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(left(), right());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+        BinaryStringFunction<?,?> other = (BinaryStringFunction<?,?>) obj;
+        return Objects.equals(other.left(), left())
+            && Objects.equals(other.right(), right());
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericFunction.java
new file mode 100644
index 0000000000000..3e3637d57df2d
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericFunction.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.type.DataType;
+
+/**
+ * A binary string function with a numeric second parameter and a string result
+ */
+public abstract class BinaryStringNumericFunction extends BinaryStringFunction<Number, String> {
+
+    public BinaryStringNumericFunction(Location location, Expression left, Expression right) {
+        super(location, left, right);
+    }
+
+    @Override
+    protected TypeResolution resolveSecondParameterInputType(DataType inputType) {
+        return inputType.isNumeric() ? 
+                TypeResolution.TYPE_RESOLVED : 
+                new TypeResolution("'%s' requires second parameter to be a numeric type, received %s", functionName(), inputType);
+    }
+    
+    @Override
+    public DataType dataType() {
+        return DataType.KEYWORD;
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessor.java
new file mode 100644
index 0000000000000..683bcba5c3f45
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessor.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringNumericProcessor.BinaryStringNumericOperation;
+
+import java.io.IOException;
+import java.util.function.BiFunction;
+
+/**
+ * Processor class covering string manipulating functions that have the first parameter as string,
+ * second parameter as numeric and a string result.
+ */
+public class BinaryStringNumericProcessor extends BinaryStringProcessor<BinaryStringNumericOperation, Number, String> {
+    
+    public static final String NAME = "sn";
+    
+    public BinaryStringNumericProcessor(StreamInput in) throws IOException {
+        super(in, i -> i.readEnum(BinaryStringNumericOperation.class));
+    }
+
+    public BinaryStringNumericProcessor(Processor left, Processor right, BinaryStringNumericOperation operation) {
+        super(left, right, operation);
+    }
+
+    public enum BinaryStringNumericOperation implements BiFunction<String, Number, String> {
+        LEFT((s,c) -> {
+            int i = c.intValue();
+            if (i < 0) return "";
+            return i > s.length() ? s : s.substring(0, i);
+        }),
+        RIGHT((s,c) -> {
+            int i = c.intValue();
+            if (i < 0) return "";
+            return i > s.length() ? s : s.substring(s.length() - i);
+        }),
+        REPEAT((s,c) -> {
+            int i = c.intValue();
+            if (i <= 0) return null;
+            
+            StringBuilder sb = new StringBuilder(s.length() * i);
+            for (int j = 0; j < i; j++) {
+                sb.append(s);
+            }
+            return sb.toString();
+        });
+
+        BinaryStringNumericOperation(BiFunction<String, Number, String> op) {
+            this.op = op;
+        }
+        
+        private final BiFunction<String, Number, String> op;
+        
+        @Override
+        public String apply(String stringExp, Number count) {
+            return op.apply(stringExp, count);
+        }
+    }
+
+    @Override
+    protected void doWrite(StreamOutput out) throws IOException {
+        out.writeEnum(operation());
+    }
+
+    @Override
+    protected Object doProcess(Object left, Object right) {
+        if (left == null || right == null) {
+            return null;
+        }
+        if (!(left instanceof String || left instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", left);
+        }
+        if (!(right instanceof Number)) {
+            throw new SqlIllegalArgumentException("A number is required; received [{}]", right);
+        }
+
+        return operation().apply(left instanceof Character ? left.toString() : (String) left, (Number) right);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorDefinition.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorDefinition.java
new file mode 100644
index 0000000000000..b0adcdf1ff2c6
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorDefinition.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.BinaryProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringNumericProcessor.BinaryStringNumericOperation;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.Objects;
+
+/**
+ * Processor definition for String operations requiring one string and one numeric argument.
+ */
+public class BinaryStringNumericProcessorDefinition extends BinaryProcessorDefinition {
+
+    private final BinaryStringNumericOperation operation;
+
+    public BinaryStringNumericProcessorDefinition(Location location, Expression expression, ProcessorDefinition left,
+            ProcessorDefinition right, BinaryStringNumericOperation operation) {
+        super(location, expression, left, right);
+        this.operation = operation;
+    }
+
+    @Override
+    protected NodeInfo<BinaryStringNumericProcessorDefinition> info() {
+        return NodeInfo.create(this, BinaryStringNumericProcessorDefinition::new, expression(), left(), right(), operation());
+    }
+
+    public BinaryStringNumericOperation operation() {
+        return operation;
+    }
+
+    @Override
+    protected BinaryProcessorDefinition replaceChildren(ProcessorDefinition newLeft, ProcessorDefinition newRight) {
+        return new BinaryStringNumericProcessorDefinition(location(), expression(), newLeft, newRight, operation());
+    }
+
+    @Override
+    public BinaryStringNumericProcessor asProcessor() {
+        return new BinaryStringNumericProcessor(left().asProcessor(), right().asProcessor(), operation());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(left(), right(), operation);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        BinaryStringNumericProcessorDefinition other = (BinaryStringNumericProcessorDefinition) obj;
+        return Objects.equals(operation, other.operation)
+                && Objects.equals(left(), other.left())
+                && Objects.equals(right(), other.right());
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringProcessor.java
new file mode 100644
index 0000000000000..0303d15adad7b
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringProcessor.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.BinaryProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.function.BiFunction;
+
+public abstract class BinaryStringProcessor<O extends Enum<?> & BiFunction<String, T, R>, T, R> extends BinaryProcessor {
+    
+    private final O operation;
+    
+    public BinaryStringProcessor(Processor left, Processor right, O operation) {
+        super(left, right);
+        this.operation = operation;
+    }
+
+    public BinaryStringProcessor(StreamInput in, Reader<O> reader) throws IOException {
+        super(in);
+        operation = reader.read(in);
+    }
+    
+    protected O operation() {
+        return operation;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(operation);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        
+        BinaryStringProcessor<?,?,?> other = (BinaryStringProcessor<?,?,?>) obj;
+        return Objects.equals(operation, other.operation)
+                && Objects.equals(left(), other.left())
+                && Objects.equals(right(), other.right());
+    }
+
+    @Override
+    public String toString() {
+        return operation.toString();
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringFunction.java
new file mode 100644
index 0000000000000..0c3c8d9453ccd
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringFunction.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.type.DataType;
+
+/**
+ * A binary string function with two string parameters and a numeric result
+ */
+public abstract class BinaryStringStringFunction extends BinaryStringFunction<String, Number> {
+
+    public BinaryStringStringFunction(Location location, Expression left, Expression right) {
+        super(location, left, right);
+    }
+
+    @Override
+    protected TypeResolution resolveSecondParameterInputType(DataType inputType) {
+        return inputType.isString() ? 
+                TypeResolution.TYPE_RESOLVED : 
+                new TypeResolution("'%s' requires second parameter to be a string type, received %s", functionName(), inputType);
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataType.INTEGER;
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessor.java
new file mode 100644
index 0000000000000..8f3425bdb4e88
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessor.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringStringProcessor.BinaryStringStringOperation;
+
+import java.io.IOException;
+import java.util.function.BiFunction;
+
+/**
+ * Processor class covering string manipulating functions that have two string parameters and a numeric result.
+ */
+public class BinaryStringStringProcessor extends BinaryStringProcessor<BinaryStringStringOperation, String, Number> {
+    
+    public static final String NAME = "ss";
+    
+    public BinaryStringStringProcessor(StreamInput in) throws IOException {
+        super(in, i -> i.readEnum(BinaryStringStringOperation.class));
+    }
+
+    public BinaryStringStringProcessor(Processor left, Processor right, BinaryStringStringOperation operation) {
+        super(left, right, operation);
+    }
+
+    public enum BinaryStringStringOperation implements BiFunction<String, String, Number> {
+        POSITION((sub,str) -> {
+            if (sub == null || str == null) return null;
+            int pos = str.indexOf(sub);
+            return pos < 0 ? 0 : pos+1;
+        });
+
+        BinaryStringStringOperation(BiFunction<String, String, Number> op) {
+            this.op = op;
+        }
+        
+        private final BiFunction<String, String, Number> op;
+        
+        @Override
+        public Number apply(String stringExpLeft, String stringExpRight) {
+            return op.apply(stringExpLeft, stringExpRight);
+        }
+    }
+
+    @Override
+    protected void doWrite(StreamOutput out) throws IOException {
+        out.writeEnum(operation());
+    }
+
+    @Override
+    protected Object doProcess(Object left, Object right) {
+        if (left == null || right == null) {
+            return null;
+        }
+        if (!(left instanceof String || left instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", left);
+        }
+        if (!(right instanceof String || right instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", right);
+        }
+
+        return operation().apply(left instanceof Character ? left.toString() : (String) left, 
+                right instanceof Character ? right.toString() : (String) right);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorDefinition.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorDefinition.java
new file mode 100644
index 0000000000000..33692845bf96e
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorDefinition.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.BinaryProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringStringProcessor.BinaryStringStringOperation;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.Objects;
+
+/**
+ * Processor definition for String operations requiring two string arguments.
+ */
+public class BinaryStringStringProcessorDefinition extends BinaryProcessorDefinition {
+
+    private final BinaryStringStringOperation operation;
+
+    public BinaryStringStringProcessorDefinition(Location location, Expression expression, ProcessorDefinition left,
+            ProcessorDefinition right, BinaryStringStringOperation operation) {
+        super(location, expression, left, right);
+        this.operation = operation;
+    }
+
+    @Override
+    protected NodeInfo<BinaryStringStringProcessorDefinition> info() {
+        return NodeInfo.create(this, BinaryStringStringProcessorDefinition::new, expression(), left(), right(), operation);
+    }
+
+    public BinaryStringStringOperation operation() {
+        return operation;
+    }
+
+    @Override
+    protected BinaryProcessorDefinition replaceChildren(ProcessorDefinition left, ProcessorDefinition right) {
+        return new BinaryStringStringProcessorDefinition(location(), expression(), left, right, operation);
+    }
+
+    @Override
+    public BinaryStringStringProcessor asProcessor() {
+        return new BinaryStringStringProcessor(left().asProcessor(), right().asProcessor(), operation);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(left(), right(), operation);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        BinaryStringStringProcessorDefinition other = (BinaryStringStringProcessorDefinition) obj;
+        return Objects.equals(operation, other.operation)
+                && Objects.equals(left(), other.left())
+                && Objects.equals(right(), other.right());
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Concat.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Concat.java
new file mode 100644
index 0000000000000..a5dc8fc9ac3d7
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Concat.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.FieldAttribute;
+import org.elasticsearch.xpack.sql.expression.function.scalar.BinaryScalarFunction;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions;
+import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+import org.elasticsearch.xpack.sql.type.DataType;
+
+import java.util.Locale;
+
+import static java.lang.String.format;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.string.ConcatFunctionProcessor.doProcessInScripts;
+
+/**
+ * Returns a string that is the result of concatenating the two strings received as parameters.
+ * The result of the function is null only if both parameters are null, otherwise the result is the non-null
+ * parameter or the concatenation of the two strings if none of them is null.
+ */
+public class Concat extends BinaryScalarFunction {
+    
+    public Concat(Location location, Expression source1, Expression source2) {
+        super(location, source1, source2);
+    }
+
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution sourceResolution = StringFunctionUtils.resolveStringInputType(left().dataType(), functionName());
+        if (sourceResolution != TypeResolution.TYPE_RESOLVED) {
+            return sourceResolution;
+        }
+
+        return StringFunctionUtils.resolveStringInputType(right().dataType(), functionName());
+    }
+
+    @Override
+    protected ProcessorDefinition makeProcessorDefinition() {
+        return new ConcatFunctionProcessorDefinition(location(), this,
+                ProcessorDefinitions.toProcessorDefinition(left()),
+                ProcessorDefinitions.toProcessorDefinition(right()));
+    }
+    
+    @Override
+    public boolean foldable() {
+        return left().foldable() 
+                && right().foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return doProcessInScripts(left().fold(), right().fold());
+    }
+    
+    @Override
+    protected Concat replaceChildren(Expression newLeft, Expression newRight) {
+        return new Concat(location(), newLeft, newRight);
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Concat::new, left(), right());
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate sourceScript1 = asScript(left());
+        ScriptTemplate sourceScript2 = asScript(right());
+
+        return asScriptFrom(sourceScript1, sourceScript2);
+    }
+
+    @Override
+    protected ScriptTemplate asScriptFrom(ScriptTemplate leftScript, ScriptTemplate rightScript) {
+        // basically, transform the script to InternalSqlScriptUtils.[function_name](function_or_field1, function_or_field2)
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{sql}.%s(%s,%s)"), 
+                "concat", 
+                leftScript.template(), 
+                rightScript.template()),
+                paramsBuilder()
+                    .script(leftScript.params()).script(rightScript.params())
+                    .build(), dataType());
+    }
+    
+    @Override
+    protected ScriptTemplate asScriptFrom(FieldAttribute field) {
+        return new ScriptTemplate(formatScript("doc[{}].value"),
+                paramsBuilder().variable(field.isInexact() ? field.exactAttribute().name() : field.name()).build(),
+                dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataType.KEYWORD;
+    }
+
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatFunctionProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatFunctionProcessor.java
new file mode 100644
index 0000000000000..3a5e9bbee24d4
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatFunctionProcessor.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.BinaryProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ConcatFunctionProcessor extends BinaryProcessor {
+
+    public static final String NAME = "cb";
+
+    public ConcatFunctionProcessor(Processor source1, Processor source2) {
+        super(source1, source2);
+    }
+    
+    public ConcatFunctionProcessor(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected Object doProcess(Object source1, Object source2) {
+        return doProcessInScripts(source1, source2);
+    }
+    
+    /**
+     * Used in Painless scripting
+     */
+    public static Object doProcessInScripts(Object source1, Object source2) {
+        if (source1 == null) {
+            return source2;
+        }
+        if (source2 == null) {
+            return source1;
+        }
+        if (!(source1 instanceof String || source1 instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", source1);
+        }
+        if (!(source2 instanceof String || source2 instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", source2);
+        }
+
+        return source1.toString().concat(source2.toString());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        
+        ConcatFunctionProcessor other = (ConcatFunctionProcessor) obj;
+        return Objects.equals(left(), other.left())
+                && Objects.equals(right(), other.right());
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(left(), right());
+    }
+
+    @Override
+    protected void doWrite(StreamOutput out) throws IOException {
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatFunctionProcessorDefinition.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatFunctionProcessorDefinition.java
new file mode 100644
index 0000000000000..f7f86419be16b
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatFunctionProcessorDefinition.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.BinaryProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.Objects;
+
+public class ConcatFunctionProcessorDefinition extends BinaryProcessorDefinition {
+
+    public ConcatFunctionProcessorDefinition(Location location, Expression expression, ProcessorDefinition left,
+            ProcessorDefinition right) {
+        super(location, expression, left, right);
+    }
+
+    @Override
+    protected NodeInfo<ConcatFunctionProcessorDefinition> info() {
+        return NodeInfo.create(this, ConcatFunctionProcessorDefinition::new, expression(), left(), right());
+    }
+
+    @Override
+    protected BinaryProcessorDefinition replaceChildren(ProcessorDefinition left, ProcessorDefinition right) {
+        return new ConcatFunctionProcessorDefinition(location(), expression(), left, right);
+    }
+
+    @Override
+    public ConcatFunctionProcessor asProcessor() {
+        return new ConcatFunctionProcessor(left().asProcessor(), right().asProcessor());
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(left(), right());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        ConcatFunctionProcessorDefinition other = (ConcatFunctionProcessorDefinition) obj;
+        return Objects.equals(left(), other.left())
+                && Objects.equals(right(), other.right());
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Insert.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Insert.java
new file mode 100644
index 0000000000000..9d635b7cdcb89
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Insert.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.FieldAttribute;
+import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions;
+import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+import org.elasticsearch.xpack.sql.type.DataType;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import static java.lang.String.format;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.string.InsertFunctionProcessor.doProcess;
+
+/**
+ * Returns a character string where length characters have been deleted from the source string, beginning at start,
+ * and where the replacement string has been inserted into the source string, beginning at start.
+ */
+public class Insert extends ScalarFunction {
+
+    private final Expression source, start, length, replacement;
+    
+    public Insert(Location location, Expression source, Expression start, Expression length, Expression replacement) {
+        super(location, Arrays.asList(source, start, length, replacement));
+        this.source = source;
+        this.start = start;
+        this.length = length;
+        this.replacement = replacement;
+    }
+    
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution sourceResolution = StringFunctionUtils.resolveStringInputType(source.dataType(), functionName());
+        if (sourceResolution != TypeResolution.TYPE_RESOLVED) {
+            return sourceResolution;
+        }
+        
+        TypeResolution startResolution = StringFunctionUtils.resolveNumericInputType(start.dataType(), functionName());
+        if (startResolution != TypeResolution.TYPE_RESOLVED) {
+            return startResolution;
+        }
+        
+        TypeResolution lengthResolution = StringFunctionUtils.resolveNumericInputType(length.dataType(), functionName());
+        if (lengthResolution != TypeResolution.TYPE_RESOLVED) {
+            return lengthResolution;
+        }
+        
+        return StringFunctionUtils.resolveStringInputType(replacement.dataType(), functionName());
+    }
+
+    @Override
+    public boolean foldable() {
+        return source.foldable() 
+                && start.foldable()
+                && length.foldable()
+                && replacement.foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(source.fold(), start.fold(), length.fold(), replacement.fold());
+    }
+
+    @Override
+    protected ProcessorDefinition makeProcessorDefinition() {
+        return new InsertFunctionProcessorDefinition(location(), this,
+                ProcessorDefinitions.toProcessorDefinition(source),
+                ProcessorDefinitions.toProcessorDefinition(start),
+                ProcessorDefinitions.toProcessorDefinition(length),
+                ProcessorDefinitions.toProcessorDefinition(replacement));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Insert::new, source, start, length, replacement);
+    }
+    
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate sourceScript = asScript(source);
+        ScriptTemplate startScript = asScript(start);
+        ScriptTemplate lengthScript = asScript(length);
+        ScriptTemplate replacementScript = asScript(replacement);
+
+        return asScriptFrom(sourceScript, startScript, lengthScript, replacementScript);
+    }
+
+    protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate startScript,
+            ScriptTemplate lengthScript, ScriptTemplate replacementScript)
+    {
+        // basically, transform the script to InternalSqlScriptUtils.[function_name](function_or_field1, function_or_field2,...)
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{sql}.%s(%s,%s,%s,%s)"), 
+                "insert", 
+                sourceScript.template(), 
+                startScript.template(),
+                lengthScript.template(),
+                replacementScript.template()),
+                paramsBuilder()
+                    .script(sourceScript.params()).script(startScript.params())
+                    .script(lengthScript.params()).script(replacementScript.params())
+                    .build(), dataType());
+    }
+    
+    @Override
+    protected ScriptTemplate asScriptFrom(FieldAttribute field) {
+        return new ScriptTemplate(formatScript("doc[{}].value"),
+                paramsBuilder().variable(field.isInexact() ? field.exactAttribute().name() : field.name()).build(),
+                dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataType.KEYWORD;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        if (newChildren.size() != 4) {
+            throw new IllegalArgumentException("expected [4] children but received [" + newChildren.size() + "]");
+        }
+
+        return new Insert(location(), newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3));
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertFunctionProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertFunctionProcessor.java
new file mode 100644
index 0000000000000..4649925614fa8
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertFunctionProcessor.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class InsertFunctionProcessor implements Processor {
+
+    private final Processor source, start, length, replacement;
+    public static final String NAME = "ins";
+
+    public InsertFunctionProcessor(Processor source, Processor start, Processor length, Processor replacement) {
+        this.source = source;
+        this.start = start;
+        this.length = length;
+        this.replacement = replacement;
+    }
+
+    public InsertFunctionProcessor(StreamInput in) throws IOException {
+        source = in.readNamedWriteable(Processor.class);
+        start = in.readNamedWriteable(Processor.class);
+        length = in.readNamedWriteable(Processor.class);
+        replacement = in.readNamedWriteable(Processor.class);
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(source());
+        out.writeNamedWriteable(start());
+        out.writeNamedWriteable(length());
+        out.writeNamedWriteable(replacement());
+    }
+
+    @Override
+    public Object process(Object input) {
+        return doProcess(source().process(input), start().process(input), length().process(input), replacement().process(input));
+    }
+
+    public static Object doProcess(Object source, Object start, Object length, Object replacement) {
+        if (source == null) {
+            return null;
+        }
+        if (!(source instanceof String || source instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", source);
+        }
+        if (replacement == null) {
+            return source;
+        }
+        if (!(replacement instanceof String || replacement instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", replacement);
+        }
+        if (start == null || length == null) {
+            return source;
+        }
+        if (!(start instanceof Number)) {
+            throw new SqlIllegalArgumentException("A number is required; received [{}]", start);
+        }
+        if (!(length instanceof Number)) {
+            throw new SqlIllegalArgumentException("A number is required; received [{}]", length);
+        }
+        if (((Number) length).intValue() < 0) {
+            throw new SqlIllegalArgumentException("A positive number is required for [length]; received [{}]", length);
+        }
+
+        int startInt = ((Number) start).intValue() - 1;
+        int realStart = startInt < 0 ? 0 : startInt;
+        
+        if (startInt > source.toString().length()) {
+            return source;
+        }
+        
+        StringBuilder sb = new StringBuilder(source.toString());
+        String replString = (replacement.toString());
+
+        return sb.replace(realStart, 
+                realStart + ((Number) length).intValue(), 
+                replString).toString();
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        
+        InsertFunctionProcessor other = (InsertFunctionProcessor) obj;
+        return Objects.equals(source(), other.source())
+                && Objects.equals(start(), other.start())
+                && Objects.equals(length(), other.length())
+                && Objects.equals(replacement(), other.replacement());
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(source(), start(), length(), replacement());
+    }
+    
+    public Processor source() {
+        return source;
+    }
+    
+    public Processor start() {
+        return start;
+    }
+    
+    public Processor length() {
+        return length;
+    }
+    
+    public Processor replacement() {
+        return replacement;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertFunctionProcessorDefinition.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertFunctionProcessorDefinition.java
new file mode 100644
index 0000000000000..ab82622788d6f
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertFunctionProcessorDefinition.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.execution.search.SqlSourceBuilder;
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class InsertFunctionProcessorDefinition extends ProcessorDefinition {
+
+    private final ProcessorDefinition source, start, length, replacement;
+
+    public InsertFunctionProcessorDefinition(Location location, Expression expression, 
+            ProcessorDefinition source, ProcessorDefinition start, 
+            ProcessorDefinition length, ProcessorDefinition replacement) {
+        super(location, expression, Arrays.asList(source, start, length, replacement));
+        this.source = source;
+        this.start = start;
+        this.length = length;
+        this.replacement = replacement;
+    }
+
+    @Override
+    public final ProcessorDefinition replaceChildren(List<ProcessorDefinition> newChildren) {
+        if (newChildren.size() != 4) {
+            throw new IllegalArgumentException("expected [4] children but received [" + newChildren.size() + "]");
+        }
+        return replaceChildren(newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3));
+    }
+    
+    @Override
+    public final ProcessorDefinition resolveAttributes(AttributeResolver resolver) {
+        ProcessorDefinition newSource = source.resolveAttributes(resolver);
+        ProcessorDefinition newStart = start.resolveAttributes(resolver);
+        ProcessorDefinition newLength = length.resolveAttributes(resolver);
+        ProcessorDefinition newReplacement = replacement.resolveAttributes(resolver);
+        if (newSource == source 
+                && newStart == start 
+                && newLength == length 
+                && newReplacement == replacement) {
+            return this;
+        }
+        return replaceChildren(newSource, newStart, newLength, newReplacement);
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return source.supportedByAggsOnlyQuery() 
+                && start.supportedByAggsOnlyQuery() 
+                && length.supportedByAggsOnlyQuery()
+                && replacement.supportedByAggsOnlyQuery();
+    }
+
+    @Override
+    public boolean resolved() {
+        return source.resolved() && start.resolved() && length.resolved() && replacement.resolved();
+    }
+    
+    protected ProcessorDefinition replaceChildren(ProcessorDefinition newSource, 
+            ProcessorDefinition newStart, 
+            ProcessorDefinition newLength,
+            ProcessorDefinition newReplacement) {
+        return new InsertFunctionProcessorDefinition(location(), expression(), newSource, newStart, newLength, newReplacement);
+    }
+
+    @Override
+    public final void collectFields(SqlSourceBuilder sourceBuilder) {
+        source.collectFields(sourceBuilder);
+        start.collectFields(sourceBuilder);
+        length.collectFields(sourceBuilder);
+        replacement.collectFields(sourceBuilder);
+    }
+
+    @Override
+    protected NodeInfo<InsertFunctionProcessorDefinition> info() {
+        return NodeInfo.create(this, InsertFunctionProcessorDefinition::new, expression(), source, start, length, replacement);
+    }
+
+    @Override
+    public InsertFunctionProcessor asProcessor() {
+        return new InsertFunctionProcessor(source.asProcessor(), start.asProcessor(), length.asProcessor(), replacement.asProcessor());
+    }
+    
+    public ProcessorDefinition source() {
+        return source;
+    }
+    
+    public ProcessorDefinition start() {
+        return start;
+    }
+    
+    public ProcessorDefinition length() {
+        return length;
+    }
+    
+    public ProcessorDefinition replacement() {
+        return replacement;
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(source, start, length, replacement);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        InsertFunctionProcessorDefinition other = (InsertFunctionProcessorDefinition) obj;
+        return Objects.equals(source, other.source)
+                && Objects.equals(start, other.start)
+                && Objects.equals(length, other.length)
+                && Objects.equals(replacement, other.replacement);
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Left.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Left.java
new file mode 100644
index 0000000000000..2e31ecf3e7b0b
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Left.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringNumericProcessor.BinaryStringNumericOperation;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.function.BiFunction;
+
+/**
+ * Returns the leftmost count characters of a string.
+ */
+public class Left extends BinaryStringNumericFunction {
+
+    public Left(Location location, Expression left, Expression right) {
+        super(location, left, right);
+    }
+
+    @Override
+    protected BiFunction<String, Number, String> operation() {
+        return BinaryStringNumericOperation.LEFT;
+    }
+
+    @Override
+    protected Left replaceChildren(Expression newLeft, Expression newRight) {
+        return new Left(location(), newLeft, newRight);
+    }
+
+    @Override
+    protected ProcessorDefinition makeProcessorDefinition() {
+        return new BinaryStringNumericProcessorDefinition(location(), this,
+                ProcessorDefinitions.toProcessorDefinition(left()),
+                ProcessorDefinitions.toProcessorDefinition(right()),
+                BinaryStringNumericOperation.LEFT);
+    }
+
+    @Override
+    protected NodeInfo<Left> info() {
+        return NodeInfo.create(this, Left::new, left(), right());
+    }
+
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Locate.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Locate.java
new file mode 100644
index 0000000000000..e33511cfbfd4a
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Locate.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.FieldAttribute;
+import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions;
+import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+import org.elasticsearch.xpack.sql.type.DataType;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import static java.lang.String.format;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.string.LocateFunctionProcessor.doProcess;
+
+/**
+ * Returns the starting position of the first occurrence of the pattern within the source string.
+ * The search for the first occurrence of the pattern begins with the first character position in the source string
+ * unless the optional argument, start, is specified. If start is specified, the search begins with the character 
+ * position indicated by the value of start. The first character position in the source string is indicated by the value 1.
+ * If the pattern is not found within the source string, the value 0 is returned.
+ */
+public class Locate extends ScalarFunction {
+
+    private final Expression pattern, source, start;
+    
+    public Locate(Location location, Expression pattern, Expression source, Expression start) {
+        super(location, start != null ? Arrays.asList(pattern, source, start) : Arrays.asList(pattern, source));
+        this.pattern = pattern;
+        this.source = source;
+        this.start = start;
+    }
+    
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution patternResolution = StringFunctionUtils.resolveStringInputType(pattern.dataType(), functionName());
+        if (patternResolution != TypeResolution.TYPE_RESOLVED) {
+            return patternResolution;
+        }
+        
+        TypeResolution sourceResolution = StringFunctionUtils.resolveStringInputType(source.dataType(), functionName());
+        if (sourceResolution != TypeResolution.TYPE_RESOLVED) {
+            return sourceResolution;
+        }
+        
+        return start == null ? TypeResolution.TYPE_RESOLVED : StringFunctionUtils.resolveNumericInputType(start.dataType(), functionName());
+    }
+
+    @Override
+    protected ProcessorDefinition makeProcessorDefinition() {
+        return new LocateFunctionProcessorDefinition(location(), this,
+            ProcessorDefinitions.toProcessorDefinition(pattern),
+            ProcessorDefinitions.toProcessorDefinition(source),
+            start == null ? null : ProcessorDefinitions.toProcessorDefinition(start));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Locate::new, pattern, source, start);
+    }
+
+    @Override
+    public boolean foldable() {
+        return pattern.foldable() 
+                && source.foldable()
+                && (start == null? true : start.foldable());
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(pattern.fold(), source.fold(), (start == null ? null : start.fold()));
+    }
+    
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate patternScript = asScript(pattern);
+        ScriptTemplate sourceScript = asScript(source);
+        ScriptTemplate startScript = start == null ? null : asScript(start);
+
+        return asScriptFrom(patternScript, sourceScript, startScript);
+    }
+
+    protected ScriptTemplate asScriptFrom(ScriptTemplate patternScript, ScriptTemplate sourceScript,
+            ScriptTemplate startScript)
+    {
+        if (start == null) {
+            return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{sql}.%s(%s,%s)"), 
+                    "locate", 
+                    patternScript.template(), 
+                    sourceScript.template()),
+                    paramsBuilder()
+                        .script(patternScript.params()).script(sourceScript.params())
+                        .build(), dataType());
+        }
+        // basically, transform the script to InternalSqlScriptUtils.[function_name](function_or_field1, function_or_field2,...)
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{sql}.%s(%s,%s,%s)"), 
+                "locate", 
+                patternScript.template(), 
+                sourceScript.template(),
+                startScript.template()),
+                paramsBuilder()
+                    .script(patternScript.params()).script(sourceScript.params())
+                    .script(startScript.params())
+                    .build(), dataType());
+    }
+    
+    @Override
+    protected ScriptTemplate asScriptFrom(FieldAttribute field) {
+        return new ScriptTemplate(formatScript("doc[{}].value"),
+                paramsBuilder().variable(field.isInexact() ? field.exactAttribute().name() : field.name()).build(),
+                dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataType.INTEGER;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        if (newChildren.size() != 3) {
+            throw new IllegalArgumentException("expected [3] children but received [" + newChildren.size() + "]");
+        }
+
+        return new Locate(location(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateFunctionProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateFunctionProcessor.java
new file mode 100644
index 0000000000000..7831fc44df29d
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateFunctionProcessor.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class LocateFunctionProcessor implements Processor {
+
+    private final Processor pattern, source, start;
+    public static final String NAME = "lc";
+
+    public LocateFunctionProcessor(Processor pattern, Processor source, Processor start) {
+        this.pattern = pattern;
+        this.source = source;
+        this.start = start;
+    }
+
+    public LocateFunctionProcessor(StreamInput in) throws IOException {
+        pattern = in.readNamedWriteable(Processor.class);
+        source = in.readNamedWriteable(Processor.class);
+        start = in.readOptionalNamedWriteable(Processor.class);
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(pattern);
+        out.writeNamedWriteable(source);
+        out.writeOptionalNamedWriteable(start);
+    }
+
+    @Override
+    public Object process(Object input) {
+        return doProcess(pattern().process(input), source().process(input), start() == null ? null : start().process(input));
+    }
+
+    public static Object doProcess(Object pattern, Object source, Object start) {
+        if (source == null) {
+            return null;
+        }
+        if (!(source instanceof String || source instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", source);
+        }
+        if (pattern == null) {
+            return 0;
+        }
+        
+        if (!(pattern instanceof String || pattern instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", pattern);
+        }
+        if (start != null && !(start instanceof Number)) {
+            throw new SqlIllegalArgumentException("A number is required; received [{}]", start);
+        }
+        
+        String stringSource = source instanceof Character ? source.toString() : (String) source;
+        String stringPattern = pattern instanceof Character ? pattern.toString() : (String) pattern;
+
+        return (Integer) (1 + (start != null ? 
+                stringSource.indexOf(stringPattern, ((Number) start).intValue() - 1)
+                : stringSource.indexOf(stringPattern)));
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        
+        LocateFunctionProcessor other = (LocateFunctionProcessor) obj;
+        return Objects.equals(pattern(), other.pattern())
+                && Objects.equals(source(), other.source())
+                && Objects.equals(start(), other.start());
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(pattern(), source(), start());
+    }
+    
+    public Processor pattern() {
+        return pattern;
+    }
+    
+    public Processor source() {
+        return source;
+    }
+    
+    public Processor start() {
+        return start;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateFunctionProcessorDefinition.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateFunctionProcessorDefinition.java
new file mode 100644
index 0000000000000..1902051185273
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateFunctionProcessorDefinition.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.execution.search.SqlSourceBuilder;
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class LocateFunctionProcessorDefinition extends ProcessorDefinition {
+
+    private final ProcessorDefinition pattern, source, start;
+
+    public LocateFunctionProcessorDefinition(Location location, Expression expression, ProcessorDefinition pattern,
+            ProcessorDefinition source, ProcessorDefinition start) {
+        super(location, expression, start == null ? Arrays.asList(pattern, source) : Arrays.asList(pattern, source, start));
+        this.pattern = pattern;
+        this.source = source;
+        this.start = start;
+    }
+
+    @Override
+    public final ProcessorDefinition replaceChildren(List<ProcessorDefinition> newChildren) {
+        int childrenSize = newChildren.size();
+        if (childrenSize > 3 || childrenSize < 2) {
+            throw new IllegalArgumentException("expected [2 or 3] children but received [" + newChildren.size() + "]");
+        }
+        return replaceChildren(newChildren.get(0), newChildren.get(1), childrenSize == 2 ? null : newChildren.get(2));
+    }
+
+    @Override
+    public final ProcessorDefinition resolveAttributes(AttributeResolver resolver) {
+        ProcessorDefinition newPattern = pattern.resolveAttributes(resolver);
+        ProcessorDefinition newSource = source.resolveAttributes(resolver);
+        ProcessorDefinition newStart = start == null ? start : start.resolveAttributes(resolver);
+        if (newPattern == pattern && newSource == source && newStart == start) {
+            return this;
+        }
+        return replaceChildren(newPattern, newSource, newStart);
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return pattern.supportedByAggsOnlyQuery() && source.supportedByAggsOnlyQuery()
+                && (start == null || start.supportedByAggsOnlyQuery());
+    }
+
+    @Override
+    public boolean resolved() {
+        return pattern.resolved() && source.resolved() && (start == null || start.resolved());
+    }
+
+    protected ProcessorDefinition replaceChildren(ProcessorDefinition newPattern, ProcessorDefinition newSource,
+            ProcessorDefinition newStart) {
+        return new LocateFunctionProcessorDefinition(location(), expression(), newPattern, newSource, newStart);
+    }
+
+    @Override
+    public final void collectFields(SqlSourceBuilder sourceBuilder) {
+        pattern.collectFields(sourceBuilder);
+        source.collectFields(sourceBuilder);
+        if (start != null) {
+            start.collectFields(sourceBuilder);
+        }
+    }
+
+    @Override
+    protected NodeInfo<LocateFunctionProcessorDefinition> info() {
+        return NodeInfo.create(this, LocateFunctionProcessorDefinition::new, expression(), pattern, source, start);
+    }
+
+    @Override
+    public LocateFunctionProcessor asProcessor() {
+        return new LocateFunctionProcessor(pattern.asProcessor(), source.asProcessor(), start == null ? null : start.asProcessor());
+    }
+    
+    public ProcessorDefinition source() {
+        return source;
+    }
+    
+    public ProcessorDefinition start() {
+        return start;
+    }
+    
+    public ProcessorDefinition pattern() {
+        return pattern;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(pattern, source, start);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        LocateFunctionProcessorDefinition other = (LocateFunctionProcessorDefinition) obj;
+        return Objects.equals(pattern, other.pattern) && Objects.equals(source, other.source) && Objects.equals(start, other.start);
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Position.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Position.java
new file mode 100644
index 0000000000000..9dfd3e094bb2c
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Position.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringStringProcessor.BinaryStringStringOperation;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.function.BiFunction;
+
+/**
+ * Returns the position of the first character expression in the second character expression, if not found it returns 0.
+ */
+public class Position extends BinaryStringStringFunction {
+
+    public Position(Location location, Expression left, Expression right) {
+        super(location, left, right);
+    }
+
+    @Override
+    protected BiFunction<String, String, Number> operation() {
+        return BinaryStringStringOperation.POSITION;
+    }
+
+    @Override
+    protected Position replaceChildren(Expression newLeft, Expression newRight) {
+        return new Position(location(), newLeft, newRight);
+    }
+
+    @Override
+    protected ProcessorDefinition makeProcessorDefinition() {
+        return new BinaryStringStringProcessorDefinition(location(), this,
+                ProcessorDefinitions.toProcessorDefinition(left()),
+                ProcessorDefinitions.toProcessorDefinition(right()),
+                BinaryStringStringOperation.POSITION);
+    }
+
+    @Override
+    protected NodeInfo<Position> info() {
+        return NodeInfo.create(this, Position::new, left(), right());
+    }
+
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Repeat.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Repeat.java
new file mode 100644
index 0000000000000..89ea7664e40e6
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Repeat.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringNumericProcessor.BinaryStringNumericOperation;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.function.BiFunction;
+
+/**
+ * Creates a string composed of a string repeated count times.
+ */
+public class Repeat extends BinaryStringNumericFunction {
+
+    public Repeat(Location location, Expression left, Expression right) {
+        super(location, left, right);
+    }
+
+    @Override
+    protected BiFunction<String, Number, String> operation() {
+        return BinaryStringNumericOperation.REPEAT;
+    }
+
+    @Override
+    protected Repeat replaceChildren(Expression newLeft, Expression newRight) {
+        return new Repeat(location(), newLeft, newRight);
+    }
+
+    @Override
+    protected ProcessorDefinition makeProcessorDefinition() {
+        return new BinaryStringNumericProcessorDefinition(location(), this,
+                ProcessorDefinitions.toProcessorDefinition(left()),
+                ProcessorDefinitions.toProcessorDefinition(right()),
+                BinaryStringNumericOperation.REPEAT);
+    }
+
+    @Override
+    protected NodeInfo<Repeat> info() {
+        return NodeInfo.create(this, Repeat::new, left(), right());
+    }
+
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Replace.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Replace.java
new file mode 100644
index 0000000000000..9325986ac1f1c
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Replace.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.FieldAttribute;
+import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions;
+import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+import org.elasticsearch.xpack.sql.type.DataType;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import static java.lang.String.format;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.string.SubstringFunctionProcessor.doProcess;
+
+/**
+ * Search the source string for occurrences of the pattern, and replace with the replacement string.
+ */
+public class Replace extends ScalarFunction {
+
+    private final Expression source, pattern, replacement;
+    
+    public Replace(Location location, Expression source, Expression pattern, Expression replacement) {
+        super(location, Arrays.asList(source, pattern, replacement));
+        this.source = source;
+        this.pattern = pattern;
+        this.replacement = replacement;
+    }
+    
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution sourceResolution = StringFunctionUtils.resolveStringInputType(source.dataType(), functionName());
+        if (sourceResolution != TypeResolution.TYPE_RESOLVED) {
+            return sourceResolution;
+        }
+        
+        TypeResolution patternResolution = StringFunctionUtils.resolveStringInputType(pattern.dataType(), functionName());
+        if (patternResolution != TypeResolution.TYPE_RESOLVED) {
+            return patternResolution;
+        }
+        
+        return StringFunctionUtils.resolveStringInputType(replacement.dataType(), functionName());
+    }
+
+    @Override
+    protected ProcessorDefinition makeProcessorDefinition() {
+        return new ReplaceFunctionProcessorDefinition(location(), this,
+                ProcessorDefinitions.toProcessorDefinition(source),
+                ProcessorDefinitions.toProcessorDefinition(pattern),
+                ProcessorDefinitions.toProcessorDefinition(replacement));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Replace::new, source, pattern, replacement);
+    }
+
+    @Override
+    public boolean foldable() {
+        return source.foldable() 
+                && pattern.foldable()
+                && replacement.foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(source.fold(), pattern.fold(), replacement.fold());
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate sourceScript = asScript(source);
+        ScriptTemplate patternScript = asScript(pattern);
+        ScriptTemplate replacementScript = asScript(replacement);
+
+        return asScriptFrom(sourceScript, patternScript, replacementScript);
+    }
+
+    protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate patternScript,
+            ScriptTemplate replacementScript)
+    {
+        // basically, transform the script to InternalSqlScriptUtils.[function_name](function_or_field1, function_or_field2,...)
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{sql}.%s(%s,%s,%s)"), 
+                "replace", 
+                sourceScript.template(), 
+                patternScript.template(),
+                replacementScript.template()),
+                paramsBuilder()
+                    .script(sourceScript.params()).script(patternScript.params())
+                    .script(replacementScript.params())
+                    .build(), dataType());
+    }
+    
+    @Override
+    protected ScriptTemplate asScriptFrom(FieldAttribute field) {
+        return new ScriptTemplate(formatScript("doc[{}].value"),
+                paramsBuilder().variable(field.isInexact() ? field.exactAttribute().name() : field.name()).build(),
+                dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataType.KEYWORD;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        if (newChildren.size() != 3) {
+            throw new IllegalArgumentException("expected [3] children but received [" + newChildren.size() + "]");
+        }
+
+        return new Replace(location(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceFunctionProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceFunctionProcessor.java
new file mode 100644
index 0000000000000..529ceb408aaa6
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceFunctionProcessor.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ReplaceFunctionProcessor implements Processor {
+
+    private final Processor source, pattern, replacement;
+    public static final String NAME = "r";
+
+    public ReplaceFunctionProcessor(Processor source, Processor pattern, Processor replacement) {
+        this.source = source;
+        this.pattern = pattern;
+        this.replacement = replacement;
+    }
+
+    public ReplaceFunctionProcessor(StreamInput in) throws IOException {
+        source = in.readNamedWriteable(Processor.class);
+        pattern = in.readNamedWriteable(Processor.class);
+        replacement = in.readNamedWriteable(Processor.class);
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(source);
+        out.writeNamedWriteable(pattern);
+        out.writeNamedWriteable(replacement);
+    }
+
+    @Override
+    public Object process(Object input) {
+        return doProcess(source().process(input), pattern().process(input), replacement().process(input));
+    }
+
+    public static Object doProcess(Object source, Object pattern, Object replacement) {
+        if (source == null) {
+            return null;
+        }
+        if (!(source instanceof String || source instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", source);
+        }
+        if (pattern == null || replacement == null) {
+            return source;
+        }
+        if (!(pattern instanceof String || pattern instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", pattern);
+        }
+        if (!(replacement instanceof String || replacement instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", replacement);
+        }
+        
+        return Strings.replace(source instanceof Character ? source.toString() : (String)source, 
+                pattern instanceof Character ? pattern.toString() : (String) pattern, 
+                replacement instanceof Character ? replacement.toString() : (String) replacement);
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        
+        ReplaceFunctionProcessor other = (ReplaceFunctionProcessor) obj;
+        return Objects.equals(source(), other.source())
+                && Objects.equals(pattern(), other.pattern())
+                && Objects.equals(replacement(), other.replacement());
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(source(), pattern(), replacement());
+    }
+    
+    public Processor source() {
+        return source;
+    }
+
+    public Processor pattern() {
+        return pattern;
+    }
+    
+    public Processor replacement() {
+        return replacement;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceFunctionProcessorDefinition.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceFunctionProcessorDefinition.java
new file mode 100644
index 0000000000000..14e7e929e3555
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceFunctionProcessorDefinition.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.execution.search.SqlSourceBuilder;
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class ReplaceFunctionProcessorDefinition extends ProcessorDefinition {
+
+    private final ProcessorDefinition source, pattern, replacement;
+
+    public ReplaceFunctionProcessorDefinition(Location location, Expression expression, ProcessorDefinition source,
+            ProcessorDefinition pattern, ProcessorDefinition replacement) {
+        super(location, expression, Arrays.asList(source, pattern, replacement));
+        this.source = source;
+        this.pattern = pattern;
+        this.replacement = replacement;
+    }
+
+    @Override
+    public final ProcessorDefinition replaceChildren(List<ProcessorDefinition> newChildren) {
+        if (newChildren.size() != 3) {
+            throw new IllegalArgumentException("expected [3] children but received [" + newChildren.size() + "]");
+        }
+        return replaceChildren(newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+    
+    @Override
+    public final ProcessorDefinition resolveAttributes(AttributeResolver resolver) {
+        ProcessorDefinition newSource = source.resolveAttributes(resolver);
+        ProcessorDefinition newPattern = pattern.resolveAttributes(resolver);
+        ProcessorDefinition newReplacement = replacement.resolveAttributes(resolver);
+        if (newSource == source && newPattern == pattern && newReplacement == replacement) {
+            return this;
+        }
+        return replaceChildren(newSource, newPattern, newReplacement);
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return source.supportedByAggsOnlyQuery() && pattern.supportedByAggsOnlyQuery() && replacement.supportedByAggsOnlyQuery();
+    }
+
+    @Override
+    public boolean resolved() {
+        return source.resolved() && pattern.resolved() && replacement.resolved();
+    }
+    
+    protected ProcessorDefinition replaceChildren(ProcessorDefinition newSource, ProcessorDefinition newPattern,
+            ProcessorDefinition newReplacement) {
+        return new ReplaceFunctionProcessorDefinition(location(), expression(), newSource, newPattern, newReplacement);
+    }
+
+    @Override
+    public final void collectFields(SqlSourceBuilder sourceBuilder) {
+        source.collectFields(sourceBuilder);
+        pattern.collectFields(sourceBuilder);
+        replacement.collectFields(sourceBuilder);
+    }
+
+    @Override
+    protected NodeInfo<ReplaceFunctionProcessorDefinition> info() {
+        return NodeInfo.create(this, ReplaceFunctionProcessorDefinition::new, expression(), source, pattern, replacement);
+    }
+
+    @Override
+    public ReplaceFunctionProcessor asProcessor() {
+        return new ReplaceFunctionProcessor(source.asProcessor(), pattern.asProcessor(), replacement.asProcessor());
+    }
+    
+    public ProcessorDefinition source() {
+        return source;
+    }
+    
+    public ProcessorDefinition pattern() {
+        return pattern;
+    }
+    
+    public ProcessorDefinition replacement() {
+        return replacement;
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(source, pattern, replacement);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        ReplaceFunctionProcessorDefinition other = (ReplaceFunctionProcessorDefinition) obj;
+        return Objects.equals(source, other.source)
+                && Objects.equals(pattern, other.pattern)
+                && Objects.equals(replacement, other.replacement);
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Right.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Right.java
new file mode 100644
index 0000000000000..3250dcc0a6e79
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Right.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringNumericProcessor.BinaryStringNumericOperation;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.function.BiFunction;
+
+/**
+ * Returns the rightmost count characters of a string.
+ */
+public class Right extends BinaryStringNumericFunction {
+
+    public Right(Location location, Expression left, Expression right) {
+        super(location, left, right);
+    }
+
+    @Override
+    protected BiFunction<String, Number, String> operation() {
+        return BinaryStringNumericOperation.RIGHT;
+    }
+
+    @Override
+    protected Right replaceChildren(Expression newLeft, Expression newRight) {
+        return new Right(location(), newLeft, newRight);
+    }
+
+    @Override
+    protected ProcessorDefinition makeProcessorDefinition() {
+        return new BinaryStringNumericProcessorDefinition(location(), this,
+                ProcessorDefinitions.toProcessorDefinition(left()),
+                ProcessorDefinitions.toProcessorDefinition(right()),
+                BinaryStringNumericOperation.RIGHT);
+    }
+
+    @Override
+    protected NodeInfo<Right> info() {
+        return NodeInfo.create(this, Right::new, left(), right());
+    }
+
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java
index cef826d37ed5c..1f38456cba131 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java
@@ -5,8 +5,33 @@
  */
 package org.elasticsearch.xpack.sql.expression.function.scalar.string;
 
+import org.elasticsearch.xpack.sql.expression.Expression.TypeResolution;
+import org.elasticsearch.xpack.sql.type.DataType;
+
 abstract class StringFunctionUtils {
 
+    /**
+     * Extract a substring from the given string, using start index and length of the extracted substring.
+     *
+     * @param s       the original String
+     * @param start   starting position for the substring within the original string. 0-based index position
+     * @param length  length in characters of the substracted substring
+     * @return the resulting String
+     */
+    static String substring(String s, int start, int length) {
+        if (!hasLength(s)) {
+            return s;
+        }
+        
+        if (start < 0)
+            start = 0;
+        
+        if (start + 1 > s.length() || length < 0)
+            return "";
+        
+        return (start + length > s.length()) ? s.substring(start) : s.substring(start, start + length);
+    }
+
     /**
      * Trims the trailing whitespace characters from the given String. Uses {@link Character#isWhitespace(char)}
      * to determine if a character is whitespace or not.
@@ -48,4 +73,16 @@ static String trimLeadingWhitespaces(String s) {
     private static boolean hasLength(String s) {
         return (s != null && s.length() > 0);
     }
+    
+    static TypeResolution resolveStringInputType(DataType inputType, String functionName) {
+        return inputType.isString() ? 
+                TypeResolution.TYPE_RESOLVED : 
+                new TypeResolution("'%s' requires a string type, received %s", functionName, inputType.esType);
+    }
+    
+    static TypeResolution resolveNumericInputType(DataType inputType, String functionName) {
+        return inputType.isNumeric() ? 
+                TypeResolution.TYPE_RESOLVED : 
+                new TypeResolution("'%s' requires a numeric type, received %s", functionName, inputType.esType);
+    }
 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Substring.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Substring.java
new file mode 100644
index 0000000000000..199cda01b9b5c
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Substring.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.FieldAttribute;
+import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions;
+import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+import org.elasticsearch.xpack.sql.type.DataType;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import static java.lang.String.format;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.string.SubstringFunctionProcessor.doProcess;
+
+/**
+ * Returns a character string that is derived from the source string, beginning at the character position specified by start 
+ * for length characters.
+ */
+public class Substring extends ScalarFunction {
+
+    private final Expression source, start, length;
+
+    public Substring(Location location, Expression source, Expression start, Expression length) {
+        super(location, Arrays.asList(source, start, length));
+        this.source = source;
+        this.start = start;
+        this.length = length;
+    }
+
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution sourceResolution = StringFunctionUtils.resolveStringInputType(source.dataType(), functionName());
+        if (sourceResolution != TypeResolution.TYPE_RESOLVED) {
+            return sourceResolution;
+        }
+
+        TypeResolution startResolution = StringFunctionUtils.resolveNumericInputType(start.dataType(), functionName());
+        if (startResolution != TypeResolution.TYPE_RESOLVED) {
+            return startResolution;
+        }
+
+        return StringFunctionUtils.resolveNumericInputType(length.dataType(), functionName());
+    }
+
+    @Override
+    protected ProcessorDefinition makeProcessorDefinition() {
+        return new SubstringFunctionProcessorDefinition(location(), this,
+                ProcessorDefinitions.toProcessorDefinition(source),
+                ProcessorDefinitions.toProcessorDefinition(start),
+                ProcessorDefinitions.toProcessorDefinition(length));
+    }
+    
+    @Override
+    public boolean foldable() {
+        return source.foldable() 
+                && start.foldable()
+                && length.foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(source.fold(), start.fold(), length.fold());
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Substring::new, source, start, length);
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate sourceScript = asScript(source);
+        ScriptTemplate startScript = asScript(start);
+        ScriptTemplate lengthScript = asScript(length);
+
+        return asScriptFrom(sourceScript, startScript, lengthScript);
+    }
+
+    protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate startScript,
+            ScriptTemplate lengthScript)
+    {
+        // basically, transform the script to InternalSqlScriptUtils.[function_name](function_or_field1, function_or_field2,...)
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{sql}.%s(%s,%s,%s)"), 
+                "substring", 
+                sourceScript.template(), 
+                startScript.template(),
+                lengthScript.template()),
+                paramsBuilder()
+                    .script(sourceScript.params()).script(startScript.params())
+                    .script(lengthScript.params())
+                    .build(), dataType());
+    }
+    
+    @Override
+    protected ScriptTemplate asScriptFrom(FieldAttribute field) {
+        return new ScriptTemplate(formatScript("doc[{}].value"),
+                paramsBuilder().variable(field.isInexact() ? field.exactAttribute().name() : field.name()).build(),
+                dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataType.KEYWORD;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        if (newChildren.size() != 3) {
+            throw new IllegalArgumentException("expected [3] children but received [" + newChildren.size() + "]");
+        }
+
+        return new Substring(location(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringFunctionProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringFunctionProcessor.java
new file mode 100644
index 0000000000000..28f251bf0dbac
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringFunctionProcessor.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class SubstringFunctionProcessor implements Processor {
+
+    private final Processor source, start, length;
+    public static final String NAME = "sb";
+
+    public SubstringFunctionProcessor(Processor source, Processor start, Processor length) {
+        this.source = source;
+        this.start = start;
+        this.length = length;
+    }
+
+    public SubstringFunctionProcessor(StreamInput in) throws IOException {
+        source = in.readNamedWriteable(Processor.class);
+        start = in.readNamedWriteable(Processor.class);
+        length = in.readNamedWriteable(Processor.class);
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(source);
+        out.writeNamedWriteable(start);
+        out.writeNamedWriteable(length);
+    }
+
+    @Override
+    public Object process(Object input) {
+        return doProcess(source.process(input), start.process(input), length.process(input));
+    }
+
+    public static Object doProcess(Object source, Object start, Object length) {
+        if (source == null) {
+            return null;
+        }
+        if (!(source instanceof String || source instanceof Character)) {
+            throw new SqlIllegalArgumentException("A string/char is required; received [{}]", source);
+        }
+        if (start == null || length == null) {
+            return source;
+        }
+        if (!(start instanceof Number)) {
+            throw new SqlIllegalArgumentException("A number is required; received [{}]", start);
+        }
+        if (!(length instanceof Number)) {
+            throw new SqlIllegalArgumentException("A number is required; received [{}]", length);
+        }
+        if (((Number) length).intValue() < 0) {
+            throw new SqlIllegalArgumentException("A positive number is required for [length]; received [{}]", length);
+        }
+
+        return StringFunctionUtils.substring(source instanceof Character ? source.toString() : (String) source, 
+                ((Number) start).intValue() - 1, // SQL is 1-based when it comes to string manipulation
+                ((Number) length).intValue());
+    }
+    
+    protected Processor source() {
+        return source;
+    }
+    
+    protected Processor start() {
+        return start;
+    }
+    
+    protected Processor length() {
+        return length;
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        
+        SubstringFunctionProcessor other = (SubstringFunctionProcessor) obj;
+        return Objects.equals(source(), other.source())
+                && Objects.equals(start(), other.start())
+                && Objects.equals(length(), other.length());
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(source(), start(), length());
+    }
+    
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringFunctionProcessorDefinition.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringFunctionProcessorDefinition.java
new file mode 100644
index 0000000000000..653aac24f9855
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringFunctionProcessorDefinition.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.execution.search.SqlSourceBuilder;
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class SubstringFunctionProcessorDefinition extends ProcessorDefinition {
+
+    private final ProcessorDefinition source, start, length;
+
+    public SubstringFunctionProcessorDefinition(Location location, Expression expression, ProcessorDefinition source,
+            ProcessorDefinition start, ProcessorDefinition length) {
+        super(location, expression, Arrays.asList(source, start, length));
+        this.source = source;
+        this.start = start;
+        this.length = length;
+    }
+
+    @Override
+    public final ProcessorDefinition replaceChildren(List<ProcessorDefinition> newChildren) {
+        if (newChildren.size() != 3) {
+            throw new IllegalArgumentException("expected [3] children but received [" + newChildren.size() + "]");
+        }
+        return replaceChildren(newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+
+    @Override
+    public final ProcessorDefinition resolveAttributes(AttributeResolver resolver) {
+        ProcessorDefinition newSource = source.resolveAttributes(resolver);
+        ProcessorDefinition newStart = start.resolveAttributes(resolver);
+        ProcessorDefinition newLength = length.resolveAttributes(resolver);
+        if (newSource == source && newStart == start && newLength == length) {
+            return this;
+        }
+        return replaceChildren(newSource, newStart, newLength);
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return source.supportedByAggsOnlyQuery() && start.supportedByAggsOnlyQuery() && length.supportedByAggsOnlyQuery();
+    }
+
+    @Override
+    public boolean resolved() {
+        return source.resolved() && start.resolved() && length.resolved();
+    }
+
+    protected ProcessorDefinition replaceChildren(ProcessorDefinition newSource, ProcessorDefinition newStart,
+            ProcessorDefinition newLength) {
+        return new SubstringFunctionProcessorDefinition(location(), expression(), newSource, newStart, newLength);
+    }
+
+    @Override
+    public final void collectFields(SqlSourceBuilder sourceBuilder) {
+        source.collectFields(sourceBuilder);
+        start.collectFields(sourceBuilder);
+        length.collectFields(sourceBuilder);
+    }
+
+    @Override
+    protected NodeInfo<SubstringFunctionProcessorDefinition> info() {
+        return NodeInfo.create(this, SubstringFunctionProcessorDefinition::new, expression(), source, start, length);
+    }
+
+    @Override
+    public SubstringFunctionProcessor asProcessor() {
+        return new SubstringFunctionProcessor(source.asProcessor(), start.asProcessor(), length.asProcessor());
+    }
+    
+    public ProcessorDefinition source() {
+        return source;
+    }
+    
+    public ProcessorDefinition start() {
+        return start;
+    }
+    
+    public ProcessorDefinition length() {
+        return length;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(source, start, length);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        SubstringFunctionProcessorDefinition other = (SubstringFunctionProcessorDefinition) obj;
+        return Objects.equals(source, other.source) && Objects.equals(start, other.start) && Objects.equals(length, other.length);
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
index ccd5c24c6412d..12faeb78b662d 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
@@ -6,7 +6,14 @@
 package org.elasticsearch.xpack.sql.expression.function.scalar.whitelist;
 
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFunction;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringNumericProcessor.BinaryStringNumericOperation;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringStringProcessor.BinaryStringStringOperation;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.ConcatFunctionProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.InsertFunctionProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.LocateFunctionProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.ReplaceFunctionProcessor;
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.SubstringFunctionProcessor;
 
 /**
  * Whitelisted class for SQL scripts.
@@ -60,4 +67,44 @@ public static String ltrim(String s) {
     public static String space(Number n) {
         return (String) StringOperation.SPACE.apply(n);
     }
+
+    public static String left(String s, int count) {
+        return BinaryStringNumericOperation.LEFT.apply(s, count);
+    }
+    
+    public static String right(String s, int count) {
+        return BinaryStringNumericOperation.RIGHT.apply(s, count);
+    }
+    
+    public static String concat(String s1, String s2) {
+        return ConcatFunctionProcessor.doProcessInScripts(s1, s2).toString();
+    }
+    
+    public static String repeat(String s, int count) {
+        return BinaryStringNumericOperation.REPEAT.apply(s, count);
+    }
+    
+    public static Integer position(String s1, String s2) {
+        return (Integer) BinaryStringStringOperation.POSITION.apply(s1, s2);
+    }
+    
+    public static String insert(String s, int start, int length, String r) {
+        return InsertFunctionProcessor.doProcess(s, start, length, r).toString();
+    }
+    
+    public static String substring(String s, int start, int length) {
+        return SubstringFunctionProcessor.doProcess(s, start, length).toString();
+    }
+    
+    public static String replace(String s1, String s2, String s3) {
+        return ReplaceFunctionProcessor.doProcess(s1, s2, s3).toString();
+    }
+    
+    public static Integer locate(String s1, String s2, Integer pos) {
+        return (Integer) LocateFunctionProcessor.doProcess(s1, s2, pos);
+    }
+    
+    public static Integer locate(String s1, String s2) {
+        return locate(s1, s2, null);
+    }
 }
diff --git a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
index 73a002c249f3c..8f86685889c55 100644
--- a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
+++ b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
@@ -19,4 +19,14 @@ class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalS
   String rtrim(String)
   String ltrim(String)
   String space(Number)
+  String left(String, int)
+  String right(String, int)
+  String concat(String, String)
+  String repeat(String, int)
+  Integer position(String, String)
+  String insert(String, int, int, String)
+  String substring(String, int, int)
+  String replace(String, String, String)
+  Integer locate(String, String)
+  Integer locate(String, String, Integer)
 }
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
index dce665a97e95d..95949070f2e51 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
@@ -55,7 +55,7 @@ public void testMissingFunction() {
     }
 
     public void testMisspelledFunction() {
-        assertEquals("1:8: Unknown function [COONT], did you mean any of [COUNT, COT]?", verify("SELECT COONT(bool) FROM test"));
+        assertEquals("1:8: Unknown function [COONT], did you mean any of [COUNT, COT, CONCAT]?", verify("SELECT COONT(bool) FROM test"));
     }
 
     public void testMissingColumnInGroupBy() {
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/FunctionTestUtils.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/FunctionTestUtils.java
new file mode 100644
index 0000000000000..c57dff15ca99a
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/FunctionTestUtils.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.sql.expression.Literal;
+
+import java.util.BitSet;
+import java.util.Iterator;
+
+import static org.elasticsearch.xpack.sql.tree.Location.EMPTY;
+
+public final class FunctionTestUtils {
+
+    public static Literal l(Object value) {
+        return Literal.of(EMPTY, value);
+    }
+
+    public static Literal randomStringLiteral() {
+        return l(ESTestCase.randomRealisticUnicodeOfLength(1024));
+    }
+    
+    public static Literal randomIntLiteral() {
+        return l(ESTestCase.randomInt());
+    }
+    
+    public static class Combinations implements Iterable<BitSet> {
+        private int n;
+        private int k;
+
+        public Combinations(int n, int k) {
+            this.n = n;
+            this.k = k;
+        }
+
+        @Override
+        public Iterator<BitSet> iterator() {
+            return new Iterator<BitSet>() {
+                BitSet bs = new BitSet(n);
+                {
+                    bs.set(0, k);
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return bs != null;
+                }
+
+                @Override
+                public BitSet next() {
+                    BitSet old = (BitSet) bs.clone();
+                    int b = bs.previousClearBit(n - 1);
+                    int b1 = bs.previousSetBit(b);
+                    if (b1 == -1)
+                        bs = null;
+                    else {
+                        bs.clear(b1);
+                        bs.set(b1 + 1, b1 + (n - b) + 1);
+                        bs.clear(b1 + (n - b) + 1, n);
+                    }
+                    return old;
+                }
+            };
+        }
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorDefinitionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorDefinitionTests.java
new file mode 100644
index 0000000000000..0b644995fc6b2
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorDefinitionTests.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.BinaryProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringNumericProcessor.BinaryStringNumericOperation;
+import org.elasticsearch.xpack.sql.tree.AbstractNodeTestCase;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.randomIntLiteral;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.randomStringLiteral;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions.toProcessorDefinition;
+import static org.elasticsearch.xpack.sql.tree.LocationTests.randomLocation;
+
+public class BinaryStringNumericProcessorDefinitionTests 
+        extends AbstractNodeTestCase<BinaryStringNumericProcessorDefinition, ProcessorDefinition> {
+
+    @Override
+    protected BinaryStringNumericProcessorDefinition randomInstance() {
+        return randomBinaryStringNumericProcessorDefinition();
+    }
+    
+    private Expression randomBinaryStringNumericExpression() {        
+        return randomBinaryStringNumericProcessorDefinition().expression();
+    }
+    
+    private BinaryStringNumericOperation randomBinaryStringNumericOperation() {        
+        return randomBinaryStringNumericProcessorDefinition().operation();
+    }
+    
+    public static BinaryStringNumericProcessorDefinition randomBinaryStringNumericProcessorDefinition() {
+        List<ProcessorDefinition> functions = new ArrayList<>();
+        functions.add(new Left(randomLocation(), randomStringLiteral(), randomIntLiteral()).makeProcessorDefinition());
+        functions.add(new Right(randomLocation(), randomStringLiteral(), randomIntLiteral()).makeProcessorDefinition());
+        functions.add(new Repeat(randomLocation(), randomStringLiteral(), randomIntLiteral()).makeProcessorDefinition());
+        
+        return (BinaryStringNumericProcessorDefinition) randomFrom(functions);
+    }
+
+    @Override
+    public void testTransform() {
+        // test transforming only the properties (location, expression, operation), 
+        // skipping the children (the two parameters of the binary function) which are tested separately
+        BinaryStringNumericProcessorDefinition b1 = randomInstance();
+        
+        Expression newExpression = randomValueOtherThan(b1.expression(), () -> randomBinaryStringNumericExpression());
+        BinaryStringNumericProcessorDefinition newB = new BinaryStringNumericProcessorDefinition(
+                b1.location(), 
+                newExpression,
+                b1.left(), 
+                b1.right(), 
+                b1.operation());
+        assertEquals(newB, b1.transformPropertiesOnly(v -> Objects.equals(v, b1.expression()) ? newExpression : v, Expression.class));
+        
+        BinaryStringNumericProcessorDefinition b2 = randomInstance();
+        BinaryStringNumericOperation newOp = randomValueOtherThan(b2.operation(), () -> randomBinaryStringNumericOperation());
+        newB = new BinaryStringNumericProcessorDefinition(
+                b2.location(), 
+                b2.expression(),
+                b2.left(), 
+                b2.right(), 
+                newOp);
+        assertEquals(newB, 
+                b2.transformPropertiesOnly(v -> Objects.equals(v, b2.operation()) ? newOp : v, BinaryStringNumericOperation.class));
+        
+        BinaryStringNumericProcessorDefinition b3 = randomInstance();
+        Location newLoc = randomValueOtherThan(b3.location(), () -> randomLocation());
+        newB = new BinaryStringNumericProcessorDefinition(
+                newLoc, 
+                b3.expression(),
+                b3.left(), 
+                b3.right(), 
+                b3.operation());
+        assertEquals(newB, 
+                b3.transformPropertiesOnly(v -> Objects.equals(v, b3.location()) ? newLoc : v, Location.class));
+    }
+
+    @Override
+    public void testReplaceChildren() {
+        BinaryStringNumericProcessorDefinition b = randomInstance();
+        ProcessorDefinition newLeft = toProcessorDefinition((Expression) randomValueOtherThan(b.left(), () -> randomStringLiteral()));
+        ProcessorDefinition newRight = toProcessorDefinition((Expression) randomValueOtherThan(b.right(), () -> randomIntLiteral()));
+        BinaryStringNumericProcessorDefinition newB = 
+                new BinaryStringNumericProcessorDefinition(b.location(), b.expression(), b.left(), b.right(), b.operation());
+        BinaryProcessorDefinition transformed = newB.replaceChildren(newLeft, b.right());
+        
+        assertEquals(transformed.left(), newLeft);
+        assertEquals(transformed.location(), b.location());
+        assertEquals(transformed.expression(), b.expression());
+        assertEquals(transformed.right(), b.right());
+        
+        transformed = newB.replaceChildren(b.left(), newRight);
+        assertEquals(transformed.left(), b.left());
+        assertEquals(transformed.location(), b.location());
+        assertEquals(transformed.expression(), b.expression());
+        assertEquals(transformed.right(), newRight);
+        
+        transformed = newB.replaceChildren(newLeft, newRight);
+        assertEquals(transformed.left(), newLeft);
+        assertEquals(transformed.location(), b.location());
+        assertEquals(transformed.expression(), b.expression());
+        assertEquals(transformed.right(), newRight);
+    }
+    
+    @Override
+    protected BinaryStringNumericProcessorDefinition mutate(BinaryStringNumericProcessorDefinition instance) {
+        List<Function<BinaryStringNumericProcessorDefinition, BinaryStringNumericProcessorDefinition>> randoms = new ArrayList<>();
+        randoms.add(f -> new BinaryStringNumericProcessorDefinition(f.location(), 
+                f.expression(), 
+                toProcessorDefinition((Expression) randomValueOtherThan(f.left(), () -> randomStringLiteral())),
+                f.right(),
+                f.operation()));
+        randoms.add(f -> new BinaryStringNumericProcessorDefinition(f.location(), 
+                f.expression(), 
+                f.left(),
+                toProcessorDefinition((Expression) randomValueOtherThan(f.right(), () -> randomIntLiteral())),
+                f.operation()));
+        randoms.add(f -> new BinaryStringNumericProcessorDefinition(f.location(), 
+                f.expression(), 
+                toProcessorDefinition((Expression) randomValueOtherThan(f.left(), () -> randomStringLiteral())),
+                toProcessorDefinition((Expression) randomValueOtherThan(f.right(), () -> randomIntLiteral())),
+                f.operation()));
+        
+        return randomFrom(randoms).apply(instance);
+    }
+
+    @Override
+    protected BinaryStringNumericProcessorDefinition copy(BinaryStringNumericProcessorDefinition instance) {
+        return new BinaryStringNumericProcessorDefinition(instance.location(),
+                instance.expression(),
+                instance.left(),
+                instance.right(),
+                instance.operation());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorTests.java
new file mode 100644
index 0000000000000..4cfc43d50b166
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorTests.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ConstantProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringNumericProcessor.BinaryStringNumericOperation;
+
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.l;
+import static org.elasticsearch.xpack.sql.tree.Location.EMPTY;
+
+public class BinaryStringNumericProcessorTests extends AbstractWireSerializingTestCase<BinaryStringNumericProcessor> {
+    
+    @Override
+    protected BinaryStringNumericProcessor createTestInstance() {
+        return new BinaryStringNumericProcessor(
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(1, 128)), 
+                new ConstantProcessor(randomInt(256)),
+                randomFrom(BinaryStringNumericOperation.values()));
+    }
+
+    @Override
+    protected Reader<BinaryStringNumericProcessor> instanceReader() {
+        return BinaryStringNumericProcessor::new;
+    }
+    
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Processors.getNamedWriteables());
+    }
+    
+    public void testLeftFunctionWithValidInput() {
+        assertEquals("foo", new Left(EMPTY, l("foo bar"), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foo bar", new Left(EMPTY, l("foo bar"), l(7)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foo bar", new Left(EMPTY, l("foo bar"), l(123)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("f", new Left(EMPTY, l('f'), l(1)).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testLeftFunctionWithEdgeCases() {
+        assertNull(new Left(EMPTY, l("foo bar"), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Left(EMPTY, l(null), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Left(EMPTY, l(null), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("", new Left(EMPTY, l("foo bar"), l(-1)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("", new Left(EMPTY, l("foo bar"), l(0)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("", new Left(EMPTY, l('f'), l(0)).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testLeftFunctionInputValidation() {
+        SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Left(EMPTY, l(5), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [5]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Left(EMPTY, l("foo bar"), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A number is required; received [baz]", siae.getMessage());
+    }
+
+    public void testRightFunctionWithValidInput() {
+        assertEquals("bar", new Right(EMPTY, l("foo bar"), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foo bar", new Right(EMPTY, l("foo bar"), l(7)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foo bar", new Right(EMPTY, l("foo bar"), l(123)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("f", new Right(EMPTY, l('f'), l(1)).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testRightFunctionWithEdgeCases() {
+        assertNull(new Right(EMPTY, l("foo bar"), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Right(EMPTY, l(null), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Right(EMPTY, l(null), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("", new Right(EMPTY, l("foo bar"), l(-1)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("", new Right(EMPTY, l("foo bar"), l(0)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("", new Right(EMPTY, l('f'), l(0)).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testRightFunctionInputValidation() {
+        SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Right(EMPTY, l(5), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [5]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Right(EMPTY, l("foo bar"), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A number is required; received [baz]", siae.getMessage());
+    }
+
+    public void testRepeatFunctionWithValidInput() {
+        assertEquals("foofoofoo", new Repeat(EMPTY, l("foo"), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foo", new Repeat(EMPTY, l("foo"), l(1)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("fff", new Repeat(EMPTY, l('f'), l(3)).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testRepeatFunctionWithEdgeCases() {
+        assertNull(new Repeat(EMPTY, l("foo"), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Repeat(EMPTY, l(null), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Repeat(EMPTY, l(null), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Repeat(EMPTY, l("foo"), l(-1)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Repeat(EMPTY, l("foo"), l(0)).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testRepeatFunctionInputsValidation() {
+        SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Repeat(EMPTY, l(5), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [5]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Repeat(EMPTY, l("foo bar"), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A number is required; received [baz]", siae.getMessage());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorDefinitionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorDefinitionTests.java
new file mode 100644
index 0000000000000..77b1329bdeaec
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorDefinitionTests.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.BinaryProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.tree.AbstractNodeTestCase;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.randomStringLiteral;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions.toProcessorDefinition;
+import static org.elasticsearch.xpack.sql.tree.LocationTests.randomLocation;
+
+public class BinaryStringStringProcessorDefinitionTests 
+        extends AbstractNodeTestCase<BinaryStringStringProcessorDefinition, ProcessorDefinition> {
+
+    @Override
+    protected BinaryStringStringProcessorDefinition randomInstance() {
+        return randomBinaryStringStringProcessorDefinition();
+    }
+    
+    private Expression randomBinaryStringStringExpression() {        
+        return randomBinaryStringStringProcessorDefinition().expression();
+    }
+    
+    public static BinaryStringStringProcessorDefinition randomBinaryStringStringProcessorDefinition() {
+        List<ProcessorDefinition> functions = new ArrayList<>();
+        functions.add(new Position(
+                randomLocation(), 
+                randomStringLiteral(), 
+                randomStringLiteral()
+                ).makeProcessorDefinition());
+        // if we decide to add DIFFERENCE(string,string) in the future, here we'd add it as well
+        return (BinaryStringStringProcessorDefinition) randomFrom(functions);
+    }
+
+    @Override
+    public void testTransform() {
+        // test transforming only the properties (location, expression), 
+        // skipping the children (the two parameters of the binary function) which are tested separately
+        BinaryStringStringProcessorDefinition b1 = randomInstance();
+        Expression newExpression = randomValueOtherThan(b1.expression(), () -> randomBinaryStringStringExpression());
+        BinaryStringStringProcessorDefinition newB = new BinaryStringStringProcessorDefinition(
+                b1.location(), 
+                newExpression,
+                b1.left(), 
+                b1.right(), 
+                b1.operation());
+        assertEquals(newB, b1.transformPropertiesOnly(v -> Objects.equals(v, b1.expression()) ? newExpression : v, Expression.class));
+        
+        BinaryStringStringProcessorDefinition b2 = randomInstance();
+        Location newLoc = randomValueOtherThan(b2.location(), () -> randomLocation());
+        newB = new BinaryStringStringProcessorDefinition(
+                newLoc, 
+                b2.expression(),
+                b2.left(), 
+                b2.right(), 
+                b2.operation());
+        assertEquals(newB, 
+                b2.transformPropertiesOnly(v -> Objects.equals(v, b2.location()) ? newLoc : v, Location.class));
+    }
+
+    @Override
+    public void testReplaceChildren() {
+        BinaryStringStringProcessorDefinition b = randomInstance();
+        ProcessorDefinition newLeft = toProcessorDefinition((Expression) randomValueOtherThan(b.left(), () -> randomStringLiteral()));
+        ProcessorDefinition newRight = toProcessorDefinition((Expression) randomValueOtherThan(b.right(), () -> randomStringLiteral()));
+        BinaryStringStringProcessorDefinition newB = 
+                new BinaryStringStringProcessorDefinition(b.location(), b.expression(), b.left(), b.right(), b.operation());
+        
+        BinaryProcessorDefinition transformed = newB.replaceChildren(newLeft, b.right());
+        assertEquals(transformed.left(), newLeft);
+        assertEquals(transformed.location(), b.location());
+        assertEquals(transformed.expression(), b.expression());
+        assertEquals(transformed.right(), b.right());
+        
+        transformed = newB.replaceChildren(b.left(), newRight);
+        assertEquals(transformed.left(), b.left());
+        assertEquals(transformed.location(), b.location());
+        assertEquals(transformed.expression(), b.expression());
+        assertEquals(transformed.right(), newRight);
+        
+        transformed = newB.replaceChildren(newLeft, newRight);
+        assertEquals(transformed.left(), newLeft);
+        assertEquals(transformed.location(), b.location());
+        assertEquals(transformed.expression(), b.expression());
+        assertEquals(transformed.right(), newRight);
+    }
+
+    @Override
+    protected BinaryStringStringProcessorDefinition mutate(BinaryStringStringProcessorDefinition instance) {
+        List<Function<BinaryStringStringProcessorDefinition, BinaryStringStringProcessorDefinition>> randoms = new ArrayList<>();
+        randoms.add(f -> new BinaryStringStringProcessorDefinition(f.location(), 
+                f.expression(), 
+                toProcessorDefinition((Expression) randomValueOtherThan(f.left(), () -> randomStringLiteral())),
+                f.right(),
+                f.operation()));
+        randoms.add(f -> new BinaryStringStringProcessorDefinition(f.location(), 
+                f.expression(), 
+                f.left(),
+                toProcessorDefinition((Expression) randomValueOtherThan(f.right(), () -> randomStringLiteral())),
+                f.operation()));
+        randoms.add(f -> new BinaryStringStringProcessorDefinition(f.location(), 
+                f.expression(), 
+                toProcessorDefinition((Expression) randomValueOtherThan(f.left(), () -> randomStringLiteral())),
+                toProcessorDefinition((Expression) randomValueOtherThan(f.right(), () -> randomStringLiteral())),
+                f.operation()));
+        
+        return randomFrom(randoms).apply(instance);
+    }
+
+    @Override
+    protected BinaryStringStringProcessorDefinition copy(BinaryStringStringProcessorDefinition instance) {
+        return new BinaryStringStringProcessorDefinition(instance.location(),
+                instance.expression(),
+                instance.left(),
+                instance.right(),
+                instance.operation());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorTests.java
new file mode 100644
index 0000000000000..3d0a1ebb1abbb
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorTests.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ConstantProcessor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.BinaryStringStringProcessor.BinaryStringStringOperation;
+
+import static org.elasticsearch.xpack.sql.tree.Location.EMPTY;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.l;
+
+public class BinaryStringStringProcessorTests extends AbstractWireSerializingTestCase<BinaryStringStringProcessor> {
+    
+    @Override
+    protected BinaryStringStringProcessor createTestInstance() {
+        return new BinaryStringStringProcessor(
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(1, 128)), 
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(1, 128)),
+                randomFrom(BinaryStringStringOperation.values()));
+    }
+
+    @Override
+    protected Reader<BinaryStringStringProcessor> instanceReader() {
+        return BinaryStringStringProcessor::new;
+    }
+    
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Processors.getNamedWriteables());
+    }
+    
+    public void testPositionFunctionWithValidInput() {
+        assertEquals(4, new Position(EMPTY, l("bar"), l("foobar")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals(1, new Position(EMPTY, l("foo"), l("foobar")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals(0, new Position(EMPTY, l("foo"), l("bar")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals(3, new Position(EMPTY, l('r'), l("bar")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals(0, new Position(EMPTY, l('z'), l("bar")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals(1, new Position(EMPTY, l('b'), l('b')).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testPositionFunctionWithEdgeCases() {
+        assertNull(new Position(EMPTY, l("foo"), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Position(EMPTY, l(null), l("foo")).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Position(EMPTY, l(null), l(null)).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testPositionFunctionInputsValidation() {
+        SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Position(EMPTY, l(5), l("foo")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [5]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Position(EMPTY, l("foo bar"), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [3]", siae.getMessage());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatFunctionProcessorDefinitionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatFunctionProcessorDefinitionTests.java
new file mode 100644
index 0000000000000..4e14e15c1953d
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatFunctionProcessorDefinitionTests.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.BinaryProcessorDefinition;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.tree.AbstractNodeTestCase;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.randomStringLiteral;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions.toProcessorDefinition;
+import static org.elasticsearch.xpack.sql.tree.LocationTests.randomLocation;
+
+public class ConcatFunctionProcessorDefinitionTests extends AbstractNodeTestCase<ConcatFunctionProcessorDefinition, ProcessorDefinition> {
+
+    @Override
+    protected ConcatFunctionProcessorDefinition randomInstance() {
+        return randomConcatFunctionProcessorDefinition();
+    }
+    
+    private Expression randomConcatFunctionExpression() {        
+        return randomConcatFunctionProcessorDefinition().expression();
+    }
+    
+    public static ConcatFunctionProcessorDefinition randomConcatFunctionProcessorDefinition() {
+        return (ConcatFunctionProcessorDefinition) new Concat(
+                randomLocation(), 
+                randomStringLiteral(), 
+                randomStringLiteral())
+                .makeProcessorDefinition();
+    }
+
+    @Override
+    public void testTransform() {
+        // test transforming only the properties (location, expression), 
+        // skipping the children (the two parameters of the binary function) which are tested separately
+        ConcatFunctionProcessorDefinition b1 = randomInstance();
+        
+        Expression newExpression = randomValueOtherThan(b1.expression(), () -> randomConcatFunctionExpression());
+        ConcatFunctionProcessorDefinition newB = new ConcatFunctionProcessorDefinition(
+                b1.location(), 
+                newExpression,
+                b1.left(), 
+                b1.right());
+        assertEquals(newB, b1.transformPropertiesOnly(v -> Objects.equals(v, b1.expression()) ? newExpression : v, Expression.class));
+        
+        ConcatFunctionProcessorDefinition b2 = randomInstance();
+        Location newLoc = randomValueOtherThan(b2.location(), () -> randomLocation());
+        newB = new ConcatFunctionProcessorDefinition(
+                newLoc, 
+                b2.expression(),
+                b2.left(), 
+                b2.right());
+        assertEquals(newB, 
+                b2.transformPropertiesOnly(v -> Objects.equals(v, b2.location()) ? newLoc : v, Location.class));
+    }
+
+    @Override
+    public void testReplaceChildren() {
+        ConcatFunctionProcessorDefinition b = randomInstance();
+        ProcessorDefinition newLeft = toProcessorDefinition((Expression) randomValueOtherThan(b.left(), () -> randomStringLiteral()));
+        ProcessorDefinition newRight = toProcessorDefinition((Expression) randomValueOtherThan(b.right(), () -> randomStringLiteral()));
+        ConcatFunctionProcessorDefinition newB = 
+                new ConcatFunctionProcessorDefinition(b.location(), b.expression(), b.left(), b.right());
+        BinaryProcessorDefinition transformed = newB.replaceChildren(newLeft, b.right());
+        
+        assertEquals(transformed.left(), newLeft);
+        assertEquals(transformed.location(), b.location());
+        assertEquals(transformed.expression(), b.expression());
+        assertEquals(transformed.right(), b.right());
+        
+        transformed = newB.replaceChildren(b.left(), newRight);
+        assertEquals(transformed.left(), b.left());
+        assertEquals(transformed.location(), b.location());
+        assertEquals(transformed.expression(), b.expression());
+        assertEquals(transformed.right(), newRight);
+        
+        transformed = newB.replaceChildren(newLeft, newRight);
+        assertEquals(transformed.left(), newLeft);
+        assertEquals(transformed.location(), b.location());
+        assertEquals(transformed.expression(), b.expression());
+        assertEquals(transformed.right(), newRight);
+    }
+
+    @Override
+    protected ConcatFunctionProcessorDefinition mutate(ConcatFunctionProcessorDefinition instance) {
+        List<Function<ConcatFunctionProcessorDefinition, ConcatFunctionProcessorDefinition>> randoms = new ArrayList<>();
+        randoms.add(f -> new ConcatFunctionProcessorDefinition(f.location(),
+                f.expression(), 
+                toProcessorDefinition((Expression) randomValueOtherThan(f.left(), () -> randomStringLiteral())),
+                f.right()));
+        randoms.add(f -> new ConcatFunctionProcessorDefinition(f.location(),
+                f.expression(), 
+                f.left(),
+                toProcessorDefinition((Expression) randomValueOtherThan(f.right(), () -> randomStringLiteral()))));
+        randoms.add(f -> new ConcatFunctionProcessorDefinition(f.location(),
+                f.expression(), 
+                toProcessorDefinition((Expression) randomValueOtherThan(f.left(), () -> randomStringLiteral())),
+                toProcessorDefinition((Expression) randomValueOtherThan(f.right(), () -> randomStringLiteral()))));
+        
+        return randomFrom(randoms).apply(instance);
+    }
+
+    @Override
+    protected ConcatFunctionProcessorDefinition copy(ConcatFunctionProcessorDefinition instance) {
+        return new ConcatFunctionProcessorDefinition(instance.location(),
+                instance.expression(),
+                instance.left(),
+                instance.right());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatProcessorTests.java
new file mode 100644
index 0000000000000..804f9a9d150b2
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatProcessorTests.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ConstantProcessor;
+
+import static org.elasticsearch.xpack.sql.tree.Location.EMPTY;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.l;
+
+public class ConcatProcessorTests extends AbstractWireSerializingTestCase<ConcatFunctionProcessor> {
+    
+    @Override
+    protected ConcatFunctionProcessor createTestInstance() {
+        return new ConcatFunctionProcessor(
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 128)), 
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 128)));
+    }
+
+    @Override
+    protected Reader<ConcatFunctionProcessor> instanceReader() {
+        return ConcatFunctionProcessor::new;
+    }
+    
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Processors.getNamedWriteables());
+    }
+    
+    public void testConcatFunctionWithValidInput() {
+        assertEquals("foobar", new Concat(EMPTY, l("foo"), l("bar")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("fb", new Concat(EMPTY, l('f'), l('b')).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testConcatFunctionWithEdgeCases() {
+        assertEquals("foo", new Concat(EMPTY, l("foo"), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("bar", new Concat(EMPTY, l(null), l("bar")).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Concat(EMPTY, l(null), l(null)).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testConcatFunctionInputsValidation() {
+        SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Concat(EMPTY, l(5), l("foo")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [5]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Concat(EMPTY, l("foo bar"), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [3]", siae.getMessage());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertFunctionProcessorDefinitionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertFunctionProcessorDefinitionTests.java
new file mode 100644
index 0000000000000..bf17962b0837c
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertFunctionProcessorDefinitionTests.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.Combinations;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.tree.AbstractNodeTestCase;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.randomIntLiteral;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.randomStringLiteral;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions.toProcessorDefinition;
+import static org.elasticsearch.xpack.sql.tree.LocationTests.randomLocation;
+
+public class InsertFunctionProcessorDefinitionTests extends AbstractNodeTestCase<InsertFunctionProcessorDefinition, ProcessorDefinition> {
+
+    @Override
+    protected InsertFunctionProcessorDefinition randomInstance() {
+        return randomInsertFunctionProcessorDefinition();
+    }
+    
+    private Expression randomInsertFunctionExpression() {        
+        return randomInsertFunctionProcessorDefinition().expression();
+    }
+    
+    public static InsertFunctionProcessorDefinition randomInsertFunctionProcessorDefinition() {
+        return (InsertFunctionProcessorDefinition) (new Insert(randomLocation(), 
+                            randomStringLiteral(), 
+                            randomIntLiteral(),
+                            randomIntLiteral(),
+                            randomStringLiteral())
+                .makeProcessorDefinition());
+    }
+
+    @Override
+    public void testTransform() {
+        // test transforming only the properties (location, expression), 
+        // skipping the children (the two parameters of the binary function) which are tested separately
+        InsertFunctionProcessorDefinition b1 = randomInstance();
+        Expression newExpression = randomValueOtherThan(b1.expression(), () -> randomInsertFunctionExpression());
+        InsertFunctionProcessorDefinition newB = new InsertFunctionProcessorDefinition(
+                b1.location(), 
+                newExpression,
+                b1.source(), 
+                b1.start(),
+                b1.length(),
+                b1.replacement());
+        assertEquals(newB, b1.transformPropertiesOnly(v -> Objects.equals(v, b1.expression()) ? newExpression : v, Expression.class));
+        
+        InsertFunctionProcessorDefinition b2 = randomInstance();
+        Location newLoc = randomValueOtherThan(b2.location(), () -> randomLocation());
+        newB = new InsertFunctionProcessorDefinition(
+                newLoc, 
+                b2.expression(),
+                b2.source(), 
+                b2.start(),
+                b2.length(),
+                b2.replacement());
+        assertEquals(newB, 
+                b2.transformPropertiesOnly(v -> Objects.equals(v, b2.location()) ? newLoc : v, Location.class));
+    }
+
+    @Override
+    public void testReplaceChildren() {
+        InsertFunctionProcessorDefinition b = randomInstance();
+        ProcessorDefinition newSource = toProcessorDefinition((Expression) randomValueOtherThan(b.source(), () -> randomStringLiteral()));
+        ProcessorDefinition newStart = toProcessorDefinition((Expression) randomValueOtherThan(b.start(), () -> randomIntLiteral()));
+        ProcessorDefinition newLength = toProcessorDefinition((Expression) randomValueOtherThan(b.length(), () -> randomIntLiteral()));
+        ProcessorDefinition newR = toProcessorDefinition((Expression) randomValueOtherThan(b.replacement(), () -> randomStringLiteral()));
+        InsertFunctionProcessorDefinition newB = 
+                new InsertFunctionProcessorDefinition(b.location(), b.expression(), b.source(), b.start(), b.length(), b.replacement());
+        InsertFunctionProcessorDefinition transformed = null;
+        
+        // generate all the combinations of possible children modifications and test all of them
+        for(int i = 1; i < 5; i++) {
+            for(BitSet comb : new Combinations(4, i)) {
+                transformed = (InsertFunctionProcessorDefinition) newB.replaceChildren(
+                        comb.get(0) ? newSource : b.source(),
+                        comb.get(1) ? newStart : b.start(),
+                        comb.get(2) ? newLength : b.length(),
+                        comb.get(3) ? newR : b.replacement());
+                assertEquals(transformed.source(), comb.get(0) ? newSource : b.source());
+                assertEquals(transformed.start(), comb.get(1) ? newStart : b.start());
+                assertEquals(transformed.length(), comb.get(2) ? newLength : b.length());
+                assertEquals(transformed.replacement(), comb.get(3) ? newR : b.replacement());
+                assertEquals(transformed.expression(), b.expression());
+                assertEquals(transformed.location(), b.location());
+            }
+        }
+    }
+
+    @Override
+    protected InsertFunctionProcessorDefinition mutate(InsertFunctionProcessorDefinition instance) {
+        List<Function<InsertFunctionProcessorDefinition, InsertFunctionProcessorDefinition>> randoms = new ArrayList<>();
+        
+        for(int i = 1; i < 5; i++) {
+            for(BitSet comb : new Combinations(4, i)) {
+                randoms.add(f -> new InsertFunctionProcessorDefinition(
+                        f.location(),
+                        f.expression(), 
+                        comb.get(0) ? toProcessorDefinition((Expression) randomValueOtherThan(f.source(),
+                                () -> randomStringLiteral())) : f.source(),
+                        comb.get(1) ? toProcessorDefinition((Expression) randomValueOtherThan(f.start(),
+                                () -> randomIntLiteral())) : f.start(),
+                        comb.get(2) ? toProcessorDefinition((Expression) randomValueOtherThan(f.length(),
+                                () -> randomIntLiteral())): f.length(),
+                        comb.get(3) ? toProcessorDefinition((Expression) randomValueOtherThan(f.replacement(),
+                                () -> randomStringLiteral())) : f.replacement()));
+            }
+        }
+
+        return randomFrom(randoms).apply(instance);
+    }
+
+    @Override
+    protected InsertFunctionProcessorDefinition copy(InsertFunctionProcessorDefinition instance) {
+        return new InsertFunctionProcessorDefinition(instance.location(),
+                instance.expression(),
+                instance.source(),
+                instance.start(),
+                instance.length(),
+                instance.replacement());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertProcessorTests.java
new file mode 100644
index 0000000000000..851dc6f1aa8ef
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertProcessorTests.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ConstantProcessor;
+
+import static org.elasticsearch.xpack.sql.tree.Location.EMPTY;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.l;
+
+public class InsertProcessorTests extends AbstractWireSerializingTestCase<InsertFunctionProcessor> {
+    
+    @Override
+    protected InsertFunctionProcessor createTestInstance() {
+        return new InsertFunctionProcessor(
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 128)), 
+                new ConstantProcessor(randomInt(256)),
+                new ConstantProcessor(randomInt(128)),
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 256)));
+    }
+
+    @Override
+    protected Reader<InsertFunctionProcessor> instanceReader() {
+        return InsertFunctionProcessor::new;
+    }
+    
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Processors.getNamedWriteables());
+    }
+    
+    public void testInsertWithValidInputs() {
+        assertEquals("bazbar", new Insert(EMPTY, l("foobar"), l(1), l(3), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foobaz", new Insert(EMPTY, l("foobar"), l(4), l(3), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testInsertWithEdgeCases() {
+        assertNull(new Insert(EMPTY, l(null), l(4), l(3), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foobar", new Insert(EMPTY, l("foobar"), l(4), l(3), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foobar",
+                new Insert(EMPTY, l("foobar"), l(null), l(3), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foobar",
+                new Insert(EMPTY, l("foobar"), l(4), l(null), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("bazbar", new Insert(EMPTY, l("foobar"), l(-1), l(3), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foobaz", new Insert(EMPTY, l("foobar"), l(4), l(30), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foobaz", new Insert(EMPTY, l("foobar"), l(6), l(1), l('z')).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foobarbaz", 
+                new Insert(EMPTY, l("foobar"), l(7), l(1000), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foobar", 
+                new Insert(EMPTY, l("foobar"), l(8), l(1000), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("fzr", new Insert(EMPTY, l("foobar"), l(2), l(4), l('z')).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("CAR", new Insert(EMPTY, l("FOOBAR"), l(1), l(5), l("CA")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("z", new Insert(EMPTY, l('f'), l(1), l(10), l('z')).makeProcessorDefinition().asProcessor().process(null));
+        
+        assertEquals("bla", new Insert(EMPTY, l(""), l(1), l(10), l("bla")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("", new Insert(EMPTY, l(""), l(2), l(10), l("bla")).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testInsertInputsValidation() {
+        SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Insert(EMPTY, l(5), l(1), l(3), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [5]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Insert(EMPTY, l("foobar"), l(1), l(3), l(66)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [66]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Insert(EMPTY, l("foobar"), l("c"), l(3), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A number is required; received [c]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Insert(EMPTY, l("foobar"), l(1), l('z'), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A number is required; received [z]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Insert(EMPTY, l("foobar"), l(1), l(-1), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A positive number is required for [length]; received [-1]", siae.getMessage());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateFunctionProcessorDefinitionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateFunctionProcessorDefinitionTests.java
new file mode 100644
index 0000000000000..4815c9c9528b1
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateFunctionProcessorDefinitionTests.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.Combinations;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.tree.AbstractNodeTestCase;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.randomIntLiteral;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.randomStringLiteral;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions.toProcessorDefinition;
+import static org.elasticsearch.xpack.sql.tree.LocationTests.randomLocation;
+
+public class LocateFunctionProcessorDefinitionTests extends AbstractNodeTestCase<LocateFunctionProcessorDefinition, ProcessorDefinition> {
+
+    @Override
+    protected LocateFunctionProcessorDefinition randomInstance() {
+        return randomLocateFunctionProcessorDefinition();
+    }
+    
+    private Expression randomLocateFunctionExpression() {        
+        return randomLocateFunctionProcessorDefinition().expression();
+    }
+    
+    public static LocateFunctionProcessorDefinition randomLocateFunctionProcessorDefinition() {
+        return (LocateFunctionProcessorDefinition) (new Locate(randomLocation(), 
+                            randomStringLiteral(), 
+                            randomStringLiteral(),
+                            randomFrom(true, false) ? randomIntLiteral() : null)
+                .makeProcessorDefinition());
+    }
+
+    @Override
+    public void testTransform() {
+        // test transforming only the properties (location, expression), 
+        // skipping the children (the two parameters of the binary function) which are tested separately
+        LocateFunctionProcessorDefinition b1 = randomInstance();
+        Expression newExpression = randomValueOtherThan(b1.expression(), () -> randomLocateFunctionExpression());
+        LocateFunctionProcessorDefinition newB = new LocateFunctionProcessorDefinition(
+            b1.location(), 
+            newExpression,
+            b1.pattern(), 
+            b1.source(),
+            b1.start());
+
+        assertEquals(newB, b1.transformPropertiesOnly(v -> Objects.equals(v, b1.expression()) ? newExpression : v, Expression.class));
+        
+        LocateFunctionProcessorDefinition b2 = randomInstance();
+        Location newLoc = randomValueOtherThan(b2.location(), () -> randomLocation());
+        newB = new LocateFunctionProcessorDefinition(
+                newLoc, 
+                b2.expression(),
+                b2.pattern(), 
+                b2.source(),
+                b2.start());
+
+        assertEquals(newB, 
+                b2.transformPropertiesOnly(v -> Objects.equals(v, b2.location()) ? newLoc : v, Location.class));
+    }
+
+    @Override
+    public void testReplaceChildren() {
+        LocateFunctionProcessorDefinition b = randomInstance();
+        ProcessorDefinition newPattern = toProcessorDefinition((Expression) randomValueOtherThan(b.pattern(), () -> randomStringLiteral()));
+        ProcessorDefinition newSource = toProcessorDefinition((Expression) randomValueOtherThan(b.source(), () -> randomStringLiteral()));
+        ProcessorDefinition newStart;
+        
+        LocateFunctionProcessorDefinition newB = new LocateFunctionProcessorDefinition(
+                b.location(), b.expression(), b.pattern(), b.source(), b.start());
+        newStart = toProcessorDefinition((Expression) randomValueOtherThan(b.start(), () -> randomIntLiteral()));
+        LocateFunctionProcessorDefinition transformed = null;
+        
+        // generate all the combinations of possible children modifications and test all of them
+        for(int i = 1; i < 4; i++) {
+            for(BitSet comb : new Combinations(3, i)) {
+                transformed = (LocateFunctionProcessorDefinition) newB.replaceChildren(
+                        comb.get(0) ? newPattern : b.pattern(),
+                        comb.get(1) ? newSource : b.source(),
+                        comb.get(2) ? newStart : b.start());
+                
+                assertEquals(transformed.pattern(), comb.get(0) ? newPattern : b.pattern());
+                assertEquals(transformed.source(), comb.get(1) ? newSource : b.source());
+                assertEquals(transformed.start(), comb.get(2) ? newStart : b.start());
+                assertEquals(transformed.expression(), b.expression());
+                assertEquals(transformed.location(), b.location());
+            }
+        }
+    }
+
+    @Override
+    protected LocateFunctionProcessorDefinition mutate(LocateFunctionProcessorDefinition instance) {
+        List<Function<LocateFunctionProcessorDefinition, LocateFunctionProcessorDefinition>> randoms = new ArrayList<>();
+        if (instance.start() == null) {
+            for(int i = 1; i < 3; i++) {
+                for(BitSet comb : new Combinations(2, i)) {
+                    randoms.add(f -> new LocateFunctionProcessorDefinition(f.location(), 
+                            f.expression(), 
+                            comb.get(0) ? toProcessorDefinition((Expression) randomValueOtherThan(f.pattern(),
+                                    () -> randomStringLiteral())) : f.pattern(),
+                            comb.get(1) ? toProcessorDefinition((Expression) randomValueOtherThan(f.source(),
+                                    () -> randomStringLiteral())) : f.source(),
+                                    null));
+                }
+            }
+        } else {
+            for(int i = 1; i < 4; i++) {
+                for(BitSet comb : new Combinations(3, i)) {
+                    randoms.add(f -> new LocateFunctionProcessorDefinition(f.location(), 
+                            f.expression(), 
+                            comb.get(0) ? toProcessorDefinition((Expression) randomValueOtherThan(f.pattern(),
+                                    () -> randomStringLiteral())) : f.pattern(),
+                            comb.get(1) ? toProcessorDefinition((Expression) randomValueOtherThan(f.source(),
+                                    () -> randomStringLiteral())) : f.source(),
+                            comb.get(2) ? toProcessorDefinition((Expression) randomValueOtherThan(f.start(),
+                                    () -> randomIntLiteral())) : f.start()));
+                }
+            }
+        }
+        
+        return randomFrom(randoms).apply(instance);
+    }
+
+    @Override
+    protected LocateFunctionProcessorDefinition copy(LocateFunctionProcessorDefinition instance) {
+        return new LocateFunctionProcessorDefinition(instance.location(),
+                        instance.expression(),
+                        instance.pattern(),
+                        instance.source(),
+                        instance.start());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateProcessorTests.java
new file mode 100644
index 0000000000000..e03ef72b8b8ea
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateProcessorTests.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ConstantProcessor;
+
+import static org.elasticsearch.xpack.sql.tree.Location.EMPTY;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.l;
+
+public class LocateProcessorTests extends AbstractWireSerializingTestCase<LocateFunctionProcessor> {
+    
+    @Override
+    protected LocateFunctionProcessor createTestInstance() {
+        // the "start" parameter is optional and is treated as null in the constructor
+        // when it is not used. Need to take this into account when generating random
+        // values for it.
+        Integer start = frequently() ? randomInt() : null;
+        return new LocateFunctionProcessor(
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 128)), 
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 128)),
+                new ConstantProcessor(start));
+    }
+
+    @Override
+    protected Reader<LocateFunctionProcessor> instanceReader() {
+        return LocateFunctionProcessor::new;
+    }
+    
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Processors.getNamedWriteables());
+    }
+    
+    public void testLocateFunctionWithValidInput() {
+        assertEquals(4, new Locate(EMPTY, l("bar"), l("foobarbar"), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals(7, new Locate(EMPTY, l("bar"), l("foobarbar"), l(5)).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testLocateFunctionWithEdgeCasesInputs() {
+        assertEquals(4, new Locate(EMPTY, l("bar"), l("foobarbar"), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Locate(EMPTY, l("bar"), l(null), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals(0, new Locate(EMPTY, l(null), l("foobarbar"), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals(0, new Locate(EMPTY, l(null), l("foobarbar"), l(null)).makeProcessorDefinition().asProcessor().process(null));
+
+        assertEquals(1, new Locate(EMPTY, l("foo"), l("foobarbar"), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals(1, new Locate(EMPTY, l('o'), l('o'), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals(9, new Locate(EMPTY, l('r'), l("foobarbar"), l(9)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals(4, new Locate(EMPTY, l("bar"), l("foobarbar"), l(-3)).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testLocateFunctionValidatingInputs() {
+        SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Locate(EMPTY, l(5), l("foobarbar"), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [5]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Locate(EMPTY, l("foo"), l(1), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [1]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Locate(EMPTY, l("foobarbar"), l("bar"), l('c')).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A number is required; received [c]", siae.getMessage());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceFunctionProcessorDefinitionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceFunctionProcessorDefinitionTests.java
new file mode 100644
index 0000000000000..14e9e51f47107
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceFunctionProcessorDefinitionTests.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.Combinations;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.tree.AbstractNodeTestCase;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.randomStringLiteral;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions.toProcessorDefinition;
+import static org.elasticsearch.xpack.sql.tree.LocationTests.randomLocation;
+
+public class ReplaceFunctionProcessorDefinitionTests extends AbstractNodeTestCase<ReplaceFunctionProcessorDefinition, ProcessorDefinition> {
+
+    @Override
+    protected ReplaceFunctionProcessorDefinition randomInstance() {
+        return randomReplaceFunctionProcessorDefinition();
+    }
+    
+    private Expression randomReplaceFunctionExpression() {        
+        return randomReplaceFunctionProcessorDefinition().expression();
+    }
+    
+    public static ReplaceFunctionProcessorDefinition randomReplaceFunctionProcessorDefinition() {
+        return (ReplaceFunctionProcessorDefinition) (new Replace(randomLocation(), 
+                            randomStringLiteral(), 
+                            randomStringLiteral(),
+                            randomStringLiteral())
+                .makeProcessorDefinition());
+    }
+
+    @Override
+    public void testTransform() {
+        // test transforming only the properties (location, expression), 
+        // skipping the children (the two parameters of the binary function) which are tested separately
+        ReplaceFunctionProcessorDefinition b1 = randomInstance();
+        
+        Expression newExpression = randomValueOtherThan(b1.expression(), () -> randomReplaceFunctionExpression());
+        ReplaceFunctionProcessorDefinition newB = new ReplaceFunctionProcessorDefinition(
+                b1.location(), 
+                newExpression,
+                b1.source(), 
+                b1.pattern(),
+                b1.replacement());
+        assertEquals(newB, b1.transformPropertiesOnly(v -> Objects.equals(v, b1.expression()) ? newExpression : v, Expression.class));
+        
+        ReplaceFunctionProcessorDefinition b2 = randomInstance();
+        Location newLoc = randomValueOtherThan(b2.location(), () -> randomLocation());
+        newB = new ReplaceFunctionProcessorDefinition(
+                newLoc, 
+                b2.expression(),
+                b2.source(), 
+                b2.pattern(),
+                b2.replacement());
+        assertEquals(newB, 
+                b2.transformPropertiesOnly(v -> Objects.equals(v, b2.location()) ? newLoc : v, Location.class));
+    }
+
+    @Override
+    public void testReplaceChildren() {
+        ReplaceFunctionProcessorDefinition b = randomInstance();
+        ProcessorDefinition newSource = toProcessorDefinition((Expression) randomValueOtherThan(b.source(), () -> randomStringLiteral()));
+        ProcessorDefinition newPattern = toProcessorDefinition((Expression) randomValueOtherThan(b.pattern(), () -> randomStringLiteral()));
+        ProcessorDefinition newR = toProcessorDefinition((Expression) randomValueOtherThan(b.replacement(), () -> randomStringLiteral()));
+        ReplaceFunctionProcessorDefinition newB = 
+                new ReplaceFunctionProcessorDefinition(b.location(), b.expression(), b.source(), b.pattern(), b.replacement());
+        ReplaceFunctionProcessorDefinition transformed = null;
+        
+        // generate all the combinations of possible children modifications and test all of them
+        for(int i = 1; i < 4; i++) {
+            for(BitSet comb : new Combinations(3, i)) {
+                transformed = (ReplaceFunctionProcessorDefinition) newB.replaceChildren(
+                        comb.get(0) ? newSource : b.source(),
+                        comb.get(1) ? newPattern : b.pattern(),
+                        comb.get(2) ? newR : b.replacement());
+                
+                assertEquals(transformed.source(), comb.get(0) ? newSource : b.source());
+                assertEquals(transformed.pattern(), comb.get(1) ? newPattern : b.pattern());
+                assertEquals(transformed.replacement(), comb.get(2) ? newR : b.replacement());
+                assertEquals(transformed.expression(), b.expression());
+                assertEquals(transformed.location(), b.location());
+            }
+        }
+    }
+
+    @Override
+    protected ReplaceFunctionProcessorDefinition mutate(ReplaceFunctionProcessorDefinition instance) {
+        List<Function<ReplaceFunctionProcessorDefinition, ReplaceFunctionProcessorDefinition>> randoms = new ArrayList<>();
+        
+        for(int i = 1; i < 4; i++) {
+            for(BitSet comb : new Combinations(3, i)) {
+                randoms.add(f -> new ReplaceFunctionProcessorDefinition(f.location(), 
+                        f.expression(), 
+                        comb.get(0) ? toProcessorDefinition((Expression) randomValueOtherThan(f.source(),
+                                () -> randomStringLiteral())) : f.source(),
+                        comb.get(1) ? toProcessorDefinition((Expression) randomValueOtherThan(f.pattern(),
+                                () -> randomStringLiteral())) : f.pattern(),
+                        comb.get(2) ? toProcessorDefinition((Expression) randomValueOtherThan(f.replacement(),
+                                () -> randomStringLiteral())) : f.replacement()));
+            }
+        }
+        
+        return randomFrom(randoms).apply(instance);
+    }
+
+    @Override
+    protected ReplaceFunctionProcessorDefinition copy(ReplaceFunctionProcessorDefinition instance) {
+        return new ReplaceFunctionProcessorDefinition(instance.location(),
+                instance.expression(),
+                instance.source(),
+                instance.pattern(),
+                instance.replacement());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceProcessorTests.java
new file mode 100644
index 0000000000000..77c84d5e7f4a3
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceProcessorTests.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ConstantProcessor;
+
+import static org.elasticsearch.xpack.sql.tree.Location.EMPTY;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.l;
+
+public class ReplaceProcessorTests extends AbstractWireSerializingTestCase<ReplaceFunctionProcessor> {
+    
+    @Override
+    protected ReplaceFunctionProcessor createTestInstance() {
+        return new ReplaceFunctionProcessor(
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 128)), 
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 128)),
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 128)));
+    }
+
+    @Override
+    protected Reader<ReplaceFunctionProcessor> instanceReader() {
+        return ReplaceFunctionProcessor::new;
+    }
+    
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Processors.getNamedWriteables());
+    }
+        
+    public void testReplaceFunctionWithValidInput() {
+        assertEquals("foobazbaz",
+                new Replace(EMPTY, l("foobarbar"), l("bar"), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foobXrbXr", new Replace(EMPTY, l("foobarbar"), l('a'), l('X')).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("z", new Replace(EMPTY, l('f'), l('f'), l('z')).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testReplaceFunctionWithEdgeCases() {
+        assertEquals("foobarbar",
+                new Replace(EMPTY, l("foobarbar"), l("bar"), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foobarbar",
+                new Replace(EMPTY, l("foobarbar"), l(null), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Replace(EMPTY, l(null), l("bar"), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Replace(EMPTY, l(null), l(null), l(null)).makeProcessorDefinition().asProcessor().process(null));
+    }
+    
+    public void testReplaceFunctionInputsValidation() {
+        SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Replace(EMPTY, l(5), l("bar"), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [5]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Replace(EMPTY, l("foobarbar"), l(4), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [4]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Replace(EMPTY, l("foobarbar"), l("bar"), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [3]", siae.getMessage());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java
index dcfb8d278ff3f..a4d9d4cb57ab6 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation;
 
 import java.io.IOException;
+import java.util.Locale;
 
 public class StringFunctionProcessorTests extends AbstractWireSerializingTestCase<StringProcessor> {
     public static StringProcessor randomStringFunctionProcessor() {
@@ -73,6 +74,19 @@ public void testLCase() {
 
         stringCharInputValidation(proc);
     }
+    
+    public void testLCaseWithTRLocale() {
+        Locale.setDefault(Locale.forLanguageTag("tr"));
+        StringProcessor proc = new StringProcessor(StringOperation.LCASE);
+
+        // ES-SQL is not locale sensitive (so far). The obvious test for this is the Turkish language, uppercase letter I conversion
+        // in non-Turkish locale the lowercasing would create i and an additional dot, while in Turkish Locale it would only create "i"
+        // unicode 0069 = i
+        assertEquals("\u0069\u0307", proc.process("\u0130"));
+        // unicode 0049 = I (regular capital letter i)
+        // in Turkish locale this would be lowercased to a "i" without dot (unicode 0131)
+        assertEquals("\u0069", proc.process("\u0049"));
+    }
 
     public void testUCase() {
         StringProcessor proc = new StringProcessor(StringOperation.UCASE);
@@ -81,9 +95,21 @@ public void testUCase() {
         assertEquals("SOMELOWERCASE", proc.process("SomeLoweRCasE"));
         assertEquals("FULLUPPERCASE", proc.process("FULLUPPERCASE"));
         assertEquals("A", proc.process('a'));
+        
+        // special uppercasing for small letter sharp "s" resulting "SS"
+        assertEquals("\u0053\u0053", proc.process("\u00df"));
 
         stringCharInputValidation(proc);
     }
+    
+    public void testUCaseWithTRLocale() {
+        Locale.setDefault(Locale.forLanguageTag("tr"));
+        StringProcessor proc = new StringProcessor(StringOperation.UCASE);
+
+        // ES-SQL is not Locale sensitive (so far).
+        // in Turkish locale, small letter "i" is uppercased to "I" with a dot above (unicode 130), otherwise in "i" (unicode 49)
+        assertEquals("\u0049", proc.process("\u0069"));
+    }
 
     public void testLength() {
         StringProcessor proc = new StringProcessor(StringOperation.LENGTH);
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringFunctionProcessorDefinitionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringFunctionProcessorDefinitionTests.java
new file mode 100644
index 0000000000000..250949123a9b2
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringFunctionProcessorDefinitionTests.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.Combinations;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition;
+import org.elasticsearch.xpack.sql.tree.AbstractNodeTestCase;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.randomIntLiteral;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.randomStringLiteral;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions.toProcessorDefinition;
+import static org.elasticsearch.xpack.sql.tree.LocationTests.randomLocation;
+
+public class SubstringFunctionProcessorDefinitionTests 
+    extends AbstractNodeTestCase<SubstringFunctionProcessorDefinition, ProcessorDefinition> {
+
+    @Override
+    protected SubstringFunctionProcessorDefinition randomInstance() {
+        return randomSubstringFunctionProcessorDefinition();
+    }
+    
+    private Expression randomSubstringFunctionExpression() {        
+        return randomSubstringFunctionProcessorDefinition().expression();
+    }
+    
+    public static SubstringFunctionProcessorDefinition randomSubstringFunctionProcessorDefinition() {
+        return (SubstringFunctionProcessorDefinition) (new Substring(randomLocation(), 
+                            randomStringLiteral(), 
+                            randomIntLiteral(),
+                            randomIntLiteral())
+                .makeProcessorDefinition());
+    }
+
+    @Override
+    public void testTransform() {
+        // test transforming only the properties (location, expression), 
+        // skipping the children (the two parameters of the binary function) which are tested separately
+        SubstringFunctionProcessorDefinition b1 = randomInstance();
+        Expression newExpression = randomValueOtherThan(b1.expression(), () -> randomSubstringFunctionExpression());
+        SubstringFunctionProcessorDefinition newB = new SubstringFunctionProcessorDefinition(
+                b1.location(), 
+                newExpression,
+                b1.source(), 
+                b1.start(),
+                b1.length());
+        assertEquals(newB, b1.transformPropertiesOnly(v -> Objects.equals(v, b1.expression()) ? newExpression : v, Expression.class));
+        
+        SubstringFunctionProcessorDefinition b2 = randomInstance();
+        Location newLoc = randomValueOtherThan(b2.location(), () -> randomLocation());
+        newB = new SubstringFunctionProcessorDefinition(
+                newLoc, 
+                b2.expression(),
+                b2.source(), 
+                b2.start(),
+                b2.length());
+        assertEquals(newB, 
+                b2.transformPropertiesOnly(v -> Objects.equals(v, b2.location()) ? newLoc : v, Location.class));
+    }
+
+    @Override
+    public void testReplaceChildren() {
+        SubstringFunctionProcessorDefinition b = randomInstance();
+        ProcessorDefinition newSource = toProcessorDefinition((Expression) randomValueOtherThan(b.source(), () -> randomStringLiteral()));
+        ProcessorDefinition newStart = toProcessorDefinition((Expression) randomValueOtherThan(b.start(), () -> randomIntLiteral()));
+        ProcessorDefinition newLength = toProcessorDefinition((Expression) randomValueOtherThan(b.length(), () -> randomIntLiteral()));
+        SubstringFunctionProcessorDefinition newB = 
+                new SubstringFunctionProcessorDefinition(b.location(), b.expression(), b.source(), b.start(), b.length());
+        SubstringFunctionProcessorDefinition transformed = null;
+        
+        // generate all the combinations of possible children modifications and test all of them
+        for(int i = 1; i < 4; i++) {
+            for(BitSet comb : new Combinations(3, i)) {
+                transformed = (SubstringFunctionProcessorDefinition) newB.replaceChildren(
+                        comb.get(0) ? newSource : b.source(),
+                        comb.get(1) ? newStart : b.start(),
+                        comb.get(2) ? newLength : b.length());
+                assertEquals(transformed.source(), comb.get(0) ? newSource : b.source());
+                assertEquals(transformed.start(), comb.get(1) ? newStart : b.start());
+                assertEquals(transformed.length(), comb.get(2) ? newLength : b.length());
+                assertEquals(transformed.expression(), b.expression());
+                assertEquals(transformed.location(), b.location());
+            }
+        }
+    }
+
+    @Override
+    protected SubstringFunctionProcessorDefinition mutate(SubstringFunctionProcessorDefinition instance) {
+        List<Function<SubstringFunctionProcessorDefinition, SubstringFunctionProcessorDefinition>> randoms = new ArrayList<>();
+        
+        for(int i = 1; i < 4; i++) {
+            for(BitSet comb : new Combinations(3, i)) {
+                randoms.add(f -> new SubstringFunctionProcessorDefinition(
+                        f.location(),
+                        f.expression(), 
+                        comb.get(0) ? toProcessorDefinition((Expression) randomValueOtherThan(f.source(),
+                                () -> randomStringLiteral())) : f.source(),
+                        comb.get(1) ? toProcessorDefinition((Expression) randomValueOtherThan(f.start(),
+                                () -> randomIntLiteral())) : f.start(),
+                        comb.get(2) ? toProcessorDefinition((Expression) randomValueOtherThan(f.length(),
+                                () -> randomIntLiteral())): f.length()));
+            }
+        }
+        
+        return randomFrom(randoms).apply(instance);
+    }
+
+    @Override
+    protected SubstringFunctionProcessorDefinition copy(SubstringFunctionProcessorDefinition instance) {
+        return new SubstringFunctionProcessorDefinition(instance.location(),
+                instance.expression(),
+                instance.source(),
+                instance.start(),
+                instance.length());
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringProcessorTests.java
new file mode 100644
index 0000000000000..8afc63a05dac4
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringProcessorTests.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
+import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ConstantProcessor;
+
+import static org.elasticsearch.xpack.sql.tree.Location.EMPTY;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.FunctionTestUtils.l;
+
+public class SubstringProcessorTests extends AbstractWireSerializingTestCase<SubstringFunctionProcessor> {
+    
+    @Override
+    protected SubstringFunctionProcessor createTestInstance() {
+        return new SubstringFunctionProcessor(
+                new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 256)), 
+                new ConstantProcessor(randomInt(256)),
+                new ConstantProcessor(randomInt(256)));
+    }
+
+    @Override
+    protected Reader<SubstringFunctionProcessor> instanceReader() {
+        return SubstringFunctionProcessor::new;
+    }
+    
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Processors.getNamedWriteables());
+    }
+    
+    public void testSubstringFunctionWithValidInput() {
+        assertEquals("bar", new Substring(EMPTY, l("foobarbar"), l(4), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foo", new Substring(EMPTY, l("foobarbar"), l(1), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("baz", new Substring(EMPTY, l("foobarbaz"), l(7), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("f", new Substring(EMPTY, l('f'), l(1), l(1)).makeProcessorDefinition().asProcessor().process(null));
+    }
+
+    public void testSubstringFunctionWithEdgeCases() {
+        assertEquals("foobarbar",
+                new Substring(EMPTY, l("foobarbar"), l(1), l(null)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("foobarbar",
+                new Substring(EMPTY, l("foobarbar"), l(null), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Substring(EMPTY, l(null), l(1), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertNull(new Substring(EMPTY, l(null), l(null), l(null)).makeProcessorDefinition().asProcessor().process(null));
+
+        assertEquals("foo", new Substring(EMPTY, l("foobarbar"), l(-5), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("barbar", new Substring(EMPTY, l("foobarbar"), l(4), l(30)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("r", new Substring(EMPTY, l("foobarbar"), l(9), l(1)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("", new Substring(EMPTY, l("foobarbar"), l(10), l(1)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("", new Substring(EMPTY, l("foobarbar"), l(123), l(3)).makeProcessorDefinition().asProcessor().process(null));
+    }
+
+    public void testSubstringFunctionInputsValidation() {
+        SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Substring(EMPTY, l(5), l(1), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A string/char is required; received [5]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Substring(EMPTY, l("foobarbar"), l(1), l("baz")).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A number is required; received [baz]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Substring(EMPTY, l("foobarbar"), l("bar"), l(3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A number is required; received [bar]", siae.getMessage());
+        siae = expectThrows(SqlIllegalArgumentException.class,
+                () -> new Substring(EMPTY, l("foobarbar"), l(1), l(-3)).makeProcessorDefinition().asProcessor().process(null));
+        assertEquals("A positive number is required for [length]; received [-3]", siae.getMessage());
+    }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
index d172e27b2a378..298cf27fa2f9d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
@@ -70,6 +70,7 @@ setup:
             trigger_count: 0
           status:
             job_state: "stopped"
+            upgraded_doc_id: true
 
   - do:
       xpack.rollup.delete_job:
@@ -116,6 +117,7 @@ setup:
             trigger_count: 0
           status:
             job_state: "stopped"
+            upgraded_doc_id: true
 
   - do:
       xpack.rollup.delete_job:
@@ -162,6 +164,7 @@ setup:
             trigger_count: 0
           status:
             job_state: "stopped"
+            upgraded_doc_id: true
 
   - do:
       xpack.rollup.start_job:
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
index fb2d9f59e3410..f3fa8114ddbd0 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
@@ -71,6 +71,7 @@ setup:
             trigger_count: 0
           status:
             job_state: "stopped"
+            upgraded_doc_id: true
 ---
 "Test get with no jobs":
 
@@ -181,6 +182,7 @@ setup:
             trigger_count: 0
           status:
             job_state: "stopped"
+            upgraded_doc_id: true
         - config:
             id: "bar"
             index_pattern: "bar"
@@ -206,4 +208,5 @@ setup:
             trigger_count: 0
           status:
             job_state: "stopped"
+            upgraded_doc_id: true
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
index 98ef9b32e3d29..516be25be2a2d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
@@ -71,6 +71,7 @@ setup:
             trigger_count: 0
           status:
             job_state: "stopped"
+            upgraded_doc_id: true
 
 ---
 "Test put_job with existing name":
diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java
index 0972c780618bd..d91cc2a6f0148 100644
--- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java
+++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java
@@ -13,11 +13,11 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.xpack.core.upgrade.IndexUpgradeCheckVersion;
-import org.elasticsearch.xpack.core.upgrade.UpgradeActionRequired;
 
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java
index af75595d4fd85..07017e6fc0014 100644
--- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java
+++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired;
 import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.xpack.core.upgrade.UpgradeActionRequired;
 
 import java.util.HashMap;
 import java.util.List;
diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/actions/TransportIndexUpgradeInfoAction.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/actions/TransportIndexUpgradeInfoAction.java
index 67ec115d79f5c..727afe955ea4e 100644
--- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/actions/TransportIndexUpgradeInfoAction.java
+++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/actions/TransportIndexUpgradeInfoAction.java
@@ -17,17 +17,19 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoResponse;
+import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction;
 import org.elasticsearch.xpack.upgrade.IndexUpgradeService;
-import org.elasticsearch.xpack.core.upgrade.UpgradeActionRequired;
 
 import java.util.Map;
 
-public class TransportIndexUpgradeInfoAction extends TransportMasterNodeReadAction<IndexUpgradeInfoAction.Request,
-        IndexUpgradeInfoAction.Response> {
+public class TransportIndexUpgradeInfoAction
+    extends TransportMasterNodeReadAction<IndexUpgradeInfoRequest, IndexUpgradeInfoResponse> {
 
     private final IndexUpgradeService indexUpgradeService;
     private final XPackLicenseState licenseState;
@@ -40,7 +42,7 @@ public TransportIndexUpgradeInfoAction(Settings settings, TransportService trans
                                            IndexNameExpressionResolver indexNameExpressionResolver,
                                            XPackLicenseState licenseState) {
         super(settings, IndexUpgradeInfoAction.NAME, transportService, clusterService, threadPool, actionFilters,
-            IndexUpgradeInfoAction.Request::new, indexNameExpressionResolver);
+            IndexUpgradeInfoRequest::new, indexNameExpressionResolver);
         this.indexUpgradeService = indexUpgradeService;
         this.licenseState = licenseState;
     }
@@ -51,23 +53,23 @@ protected String executor() {
     }
 
     @Override
-    protected IndexUpgradeInfoAction.Response newResponse() {
-        return new IndexUpgradeInfoAction.Response();
+    protected IndexUpgradeInfoResponse newResponse() {
+        return new IndexUpgradeInfoResponse();
     }
 
     @Override
-    protected ClusterBlockException checkBlock(IndexUpgradeInfoAction.Request request, ClusterState state) {
+    protected ClusterBlockException checkBlock(IndexUpgradeInfoRequest request, ClusterState state) {
         // Cluster is not affected but we look up repositories in metadata
         return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
     }
 
     @Override
-    protected final void masterOperation(final IndexUpgradeInfoAction.Request request, ClusterState state,
-                                         final ActionListener<IndexUpgradeInfoAction.Response> listener) {
+    protected final void masterOperation(final IndexUpgradeInfoRequest request, ClusterState state,
+                                         final ActionListener<IndexUpgradeInfoResponse> listener) {
         if (licenseState.isUpgradeAllowed()) {
             Map<String, UpgradeActionRequired> results =
                     indexUpgradeService.upgradeInfo(request.indices(), request.indicesOptions(), state);
-            listener.onResponse(new IndexUpgradeInfoAction.Response(results));
+            listener.onResponse(new IndexUpgradeInfoResponse(results));
         } else {
             listener.onFailure(LicenseUtils.newComplianceException(XPackField.UPGRADE));
         }
diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/rest/RestIndexUpgradeInfoAction.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/rest/RestIndexUpgradeInfoAction.java
index 24b576187cb6e..dbb61937642ef 100644
--- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/rest/RestIndexUpgradeInfoAction.java
+++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/rest/RestIndexUpgradeInfoAction.java
@@ -9,12 +9,12 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction;
-import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction.Request;
 
 import java.io.IOException;
 
@@ -41,7 +41,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
     }
 
     private RestChannelConsumer handleGet(final RestRequest request, NodeClient client) {
-        Request infoRequest = new Request(Strings.splitStringByCommaToArray(request.param("index")));
+        IndexUpgradeInfoRequest infoRequest = new IndexUpgradeInfoRequest(Strings.splitStringByCommaToArray(request.param("index")));
         infoRequest.indicesOptions(IndicesOptions.fromRequest(request, infoRequest.indicesOptions()));
         return channel -> client.execute(IndexUpgradeInfoAction.INSTANCE, infoRequest, new RestToXContentListener<>(channel));
     }
diff --git a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java
index 870261d251546..74eb029e916e5 100644
--- a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java
+++ b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java
@@ -14,12 +14,12 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoResponse;
+import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.xpack.core.upgrade.UpgradeActionRequired;
 import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeAction;
 import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction;
-import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction.Response;
 import org.junit.Before;
 
 import java.util.Collections;
@@ -41,7 +41,7 @@ public void testIndexUpgradeInfo() {
         // Testing only negative case here, the positive test is done in bwcTests
         assertAcked(client().admin().indices().prepareCreate("test").get());
         ensureYellow("test");
-        Response response = new IndexUpgradeInfoAction.RequestBuilder(client()).setIndices("test").get();
+        IndexUpgradeInfoResponse response = new IndexUpgradeInfoAction.RequestBuilder(client()).setIndices("test").get();
         assertThat(response.getActions().entrySet(), empty());
     }
 
@@ -57,7 +57,7 @@ public void testIndexUpgradeInfoLicense() throws Exception {
                 () -> new IndexUpgradeInfoAction.RequestBuilder(client()).setIndices("test").get());
         assertThat(e.getMessage(), equalTo("current license is non-compliant for [upgrade]"));
         enableLicensing();
-        Response response = new IndexUpgradeInfoAction.RequestBuilder(client()).setIndices("test").get();
+        IndexUpgradeInfoResponse response = new IndexUpgradeInfoAction.RequestBuilder(client()).setIndices("test").get();
         assertThat(response.getActions().entrySet(), empty());
     }
 
@@ -132,7 +132,7 @@ public void testInternalUpgradePrePostChecks() throws Exception {
 
     public void testIndexUpgradeInfoOnEmptyCluster() {
         // On empty cluster asking for all indices shouldn't fail since no indices means nothing needs to be upgraded
-        Response response = new IndexUpgradeInfoAction.RequestBuilder(client()).setIndices("_all").get();
+        IndexUpgradeInfoResponse response = new IndexUpgradeInfoAction.RequestBuilder(client()).setIndices("_all").get();
         assertThat(response.getActions().entrySet(), empty());
 
         // but calling on a particular index should fail
diff --git a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeServiceTests.java b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeServiceTests.java
index 63b1c602bf9c2..5939777572b48 100644
--- a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeServiceTests.java
+++ b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeServiceTests.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.upgrade.UpgradeActionRequired;
 
 import java.io.IOException;
 import java.util.Arrays;
diff --git a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeTasksIT.java b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeTasksIT.java
index abc157119b4cf..7605af041e67c 100644
--- a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeTasksIT.java
+++ b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeTasksIT.java
@@ -30,6 +30,8 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoResponse;
+import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
@@ -39,7 +41,6 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
-import org.elasticsearch.xpack.core.upgrade.UpgradeActionRequired;
 import org.elasticsearch.xpack.core.upgrade.UpgradeField;
 import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeAction;
 import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction;
@@ -171,7 +172,7 @@ public void testParentTasksDuringUpgrade() throws Exception {
         ensureYellow("test");
 
 
-        IndexUpgradeInfoAction.Response infoResponse = new IndexUpgradeInfoAction.RequestBuilder(client()).setIndices("test").get();
+        IndexUpgradeInfoResponse infoResponse = new IndexUpgradeInfoAction.RequestBuilder(client()).setIndices("test").get();
         assertThat(infoResponse.getActions().keySet(), contains("test"));
         assertThat(infoResponse.getActions().get("test"), equalTo(UpgradeActionRequired.UPGRADE));
 
diff --git a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/actions/IndexUpgradeInfoActionRequestTests.java b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/actions/IndexUpgradeInfoActionRequestTests.java
deleted file mode 100644
index 38072755bc384..0000000000000
--- a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/actions/IndexUpgradeInfoActionRequestTests.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.upgrade.actions;
-
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction.Request;
-
-public class IndexUpgradeInfoActionRequestTests extends AbstractWireSerializingTestCase<Request> {
-    @Override
-    protected Request createTestInstance() {
-        int indexCount = randomInt(4);
-        String[] indices = new String[indexCount];
-        for (int i = 0; i < indexCount; i++) {
-            indices[i] = randomAlphaOfLength(10);
-        }
-        Request request = new Request(indices);
-        if (randomBoolean()) {
-            request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
-        }
-        return request;
-    }
-
-    @Override
-    protected Writeable.Reader<Request> instanceReader() {
-        return Request::new;
-    }
-}
diff --git a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/actions/IndexUpgradeInfoActionResponseTests.java b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/actions/IndexUpgradeInfoActionResponseTests.java
deleted file mode 100644
index 6893e45dd2e4f..0000000000000
--- a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/actions/IndexUpgradeInfoActionResponseTests.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.upgrade.actions;
-
-import org.elasticsearch.test.AbstractStreamableTestCase;
-import org.elasticsearch.xpack.core.upgrade.UpgradeActionRequired;
-import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction.Response;
-
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Map;
-
-public class IndexUpgradeInfoActionResponseTests extends AbstractStreamableTestCase<Response> {
-
-
-    @Override
-    protected Response createTestInstance() {
-        int actionsCount = randomIntBetween(0, 5);
-        Map<String, UpgradeActionRequired> actions = new HashMap<>(actionsCount);
-        for (int i = 0; i < actionsCount; i++) {
-            actions.put(randomAlphaOfLength(10), randomFrom(EnumSet.allOf(UpgradeActionRequired.class)));
-        }
-        return new Response(actions);
-    }
-
-    @Override
-    protected Response createBlankInstance() {
-        return new Response();
-    }
-}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
index 78d1d37287f60..96237b6e3deb4 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
@@ -92,12 +92,7 @@
 import org.elasticsearch.xpack.watcher.actions.webhook.WebhookAction;
 import org.elasticsearch.xpack.watcher.actions.webhook.WebhookActionFactory;
 import org.elasticsearch.xpack.watcher.common.http.HttpClient;
-import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.common.http.HttpSettings;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.condition.ArrayCompareCondition;
 import org.elasticsearch.xpack.watcher.condition.CompareCondition;
@@ -264,12 +259,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         new WatcherIndexTemplateRegistry(settings, clusterService, threadPool, client);
 
         // http client
-        Map<String, HttpAuthFactory> httpAuthFactories = new HashMap<>();
-        httpAuthFactories.put(BasicAuth.TYPE, new BasicAuthFactory(cryptoService));
-        // TODO: add more auth types, or remove this indirection
-        HttpAuthRegistry httpAuthRegistry = new HttpAuthRegistry(httpAuthFactories);
-        HttpRequestTemplate.Parser httpTemplateParser = new HttpRequestTemplate.Parser(httpAuthRegistry);
-        httpClient = new HttpClient(settings, httpAuthRegistry, getSslService());
+        httpClient = new HttpClient(settings, getSslService(), cryptoService);
 
         // notification
         EmailService emailService = new EmailService(settings, cryptoService, clusterService.getClusterSettings());
@@ -286,11 +276,9 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
 
         TextTemplateEngine templateEngine = new TextTemplateEngine(settings, scriptService);
         Map<String, EmailAttachmentParser> emailAttachmentParsers = new HashMap<>();
-        emailAttachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, httpTemplateParser,
-                templateEngine));
+        emailAttachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, templateEngine));
         emailAttachmentParsers.put(DataAttachmentParser.TYPE, new DataAttachmentParser());
-        emailAttachmentParsers.put(ReportingAttachmentParser.TYPE, new ReportingAttachmentParser(settings, httpClient, templateEngine,
-                httpAuthRegistry));
+        emailAttachmentParsers.put(ReportingAttachmentParser.TYPE, new ReportingAttachmentParser(settings, httpClient, templateEngine));
         EmailAttachmentsParser emailAttachmentsParser = new EmailAttachmentsParser(emailAttachmentParsers);
 
         // conditions
@@ -310,7 +298,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         // actions
         final Map<String, ActionFactory> actionFactoryMap = new HashMap<>();
         actionFactoryMap.put(EmailAction.TYPE, new EmailActionFactory(settings, emailService, templateEngine, emailAttachmentsParser));
-        actionFactoryMap.put(WebhookAction.TYPE, new WebhookActionFactory(settings, httpClient, httpTemplateParser, templateEngine));
+        actionFactoryMap.put(WebhookAction.TYPE, new WebhookActionFactory(settings, httpClient, templateEngine));
         actionFactoryMap.put(IndexAction.TYPE, new IndexActionFactory(settings, client));
         actionFactoryMap.put(LoggingAction.TYPE, new LoggingActionFactory(settings, templateEngine));
         actionFactoryMap.put(HipChatAction.TYPE, new HipChatActionFactory(settings, templateEngine, hipChatService));
@@ -324,7 +312,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         final Map<String, InputFactory> inputFactories = new HashMap<>();
         inputFactories.put(SearchInput.TYPE, new SearchInputFactory(settings, client, xContentRegistry, scriptService));
         inputFactories.put(SimpleInput.TYPE, new SimpleInputFactory(settings));
-        inputFactories.put(HttpInput.TYPE, new HttpInputFactory(settings, httpClient, templateEngine, httpTemplateParser));
+        inputFactories.put(HttpInput.TYPE, new HttpInputFactory(settings, httpClient, templateEngine));
         inputFactories.put(NoneInput.TYPE, new NoneInputFactory(settings));
         inputFactories.put(TransformInput.TYPE, new TransformInputFactory(settings, transformRegistry));
         final InputRegistry inputRegistry = new InputRegistry(settings, inputFactories);
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/ExecutablePagerDutyAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/ExecutablePagerDutyAction.java
index 224e72e1a3da5..59381dc33362c 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/ExecutablePagerDutyAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/ExecutablePagerDutyAction.java
@@ -47,7 +47,7 @@ public Action.Result execute(final String actionId, WatchExecutionContext ctx, P
             return new PagerDutyAction.Result.Simulated(event);
         }
 
-        SentEvent sentEvent = account.send(event, payload);
+        SentEvent sentEvent = account.send(event, payload, ctx.id().watchId());
         return new PagerDutyAction.Result.Executed(account.getName(), sentEvent);
     }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookAction.java
index 8d20d91f4dca7..febbae99d46fa 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookAction.java
@@ -55,10 +55,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return requestTemplate.toXContent(builder, params);
     }
 
-    public static WebhookAction parse(String watchId, String actionId, XContentParser parser,
-                                      HttpRequestTemplate.Parser requestParser) throws IOException {
+    public static WebhookAction parse(String watchId, String actionId, XContentParser parser) throws IOException {
         try {
-            HttpRequestTemplate request = requestParser.parse(parser);
+            HttpRequestTemplate request = HttpRequestTemplate.Parser.parse(parser);
             return new WebhookAction(request);
         } catch (ElasticsearchParseException pe) {
             throw new ElasticsearchParseException("could not parse [{}] action [{}/{}]. failed parsing http request template", pe, TYPE,
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java
index 6de16006432b7..5a7a1069932d9 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.watcher.actions.ActionFactory;
 import org.elasticsearch.xpack.watcher.common.http.HttpClient;
-import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 
 import java.io.IOException;
@@ -18,21 +17,18 @@
 public class WebhookActionFactory extends ActionFactory {
 
     private final HttpClient httpClient;
-    private final HttpRequestTemplate.Parser requestTemplateParser;
     private final TextTemplateEngine templateEngine;
 
-    public WebhookActionFactory(Settings settings, HttpClient httpClient, HttpRequestTemplate.Parser requestTemplateParser,
-                                TextTemplateEngine templateEngine) {
+    public WebhookActionFactory(Settings settings, HttpClient httpClient, TextTemplateEngine templateEngine) {
 
         super(Loggers.getLogger(ExecutableWebhookAction.class, settings));
         this.httpClient = httpClient;
-        this.requestTemplateParser = requestTemplateParser;
         this.templateEngine = templateEngine;
     }
 
     @Override
     public ExecutableWebhookAction parseExecutable(String watchId, String actionId, XContentParser parser) throws IOException {
-        return new ExecutableWebhookAction(WebhookAction.parse(watchId, actionId, parser, requestTemplateParser),
+        return new ExecutableWebhookAction(WebhookAction.parse(watchId, actionId, parser),
                 actionLogger, httpClient, templateEngine);
 
     }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuth.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/BasicAuth.java
similarity index 84%
rename from x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuth.java
rename to x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/BasicAuth.java
index 2d316735efd77..0b40df4262f91 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuth.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/BasicAuth.java
@@ -3,22 +3,22 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.watcher.common.http.auth.basic;
+package org.elasticsearch.xpack.watcher.common.http;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.watcher.common.secret.Secret;
 import org.elasticsearch.xpack.core.watcher.crypto.CryptoService;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherXContentParser;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth;
 
 import java.io.IOException;
 import java.util.Objects;
 
-public class BasicAuth implements HttpAuth {
+public class BasicAuth implements ToXContentObject {
 
     public static final String TYPE = "basic";
 
@@ -34,11 +34,6 @@ public BasicAuth(String username, Secret password) {
         this.password = password;
     }
 
-    @Override
-    public String type() {
-        return TYPE;
-    }
-
     public String getUsername() {
         return username;
     }
@@ -74,7 +69,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder.endObject();
     }
 
-    public static BasicAuth parse(XContentParser parser) throws IOException {
+    public static BasicAuth parseInner(XContentParser parser) throws IOException {
         String username = null;
         Secret password = null;
 
@@ -103,6 +98,20 @@ public static BasicAuth parse(XContentParser parser) throws IOException {
         return new BasicAuth(username, password);
     }
 
+    public static BasicAuth parse(XContentParser parser) throws IOException {
+        String type = null;
+        XContentParser.Token token;
+        BasicAuth auth = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                type = parser.currentName();
+            } else if (token == XContentParser.Token.START_OBJECT && type != null) {
+                auth = BasicAuth.parseInner(parser);
+            }
+        }
+        return auth;
+    }
+
     interface Field {
         ParseField USERNAME = new ParseField("username");
         ParseField PASSWORD = new ParseField("password");
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java
index 97d7779346fae..547e3b7197f4a 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java
@@ -10,6 +10,8 @@
 import org.apache.http.HttpHost;
 import org.apache.http.NameValuePair;
 import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.AuthCache;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.client.config.RequestConfig;
@@ -42,8 +44,7 @@
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
 import org.elasticsearch.xpack.core.ssl.SSLConfiguration;
 import org.elasticsearch.xpack.core.ssl.SSLService;
-import org.elasticsearch.xpack.watcher.common.http.auth.ApplicableHttpAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
+import org.elasticsearch.xpack.core.watcher.crypto.CryptoService;
 
 import javax.net.ssl.HostnameVerifier;
 import java.io.ByteArrayOutputStream;
@@ -66,20 +67,20 @@ public class HttpClient extends AbstractComponent implements Closeable {
     // you are querying a remote Elasticsearch cluster
     private static final int MAX_CONNECTIONS = 500;
 
-    private final HttpAuthRegistry httpAuthRegistry;
     private final CloseableHttpClient client;
     private final HttpProxy settingsProxy;
     private final TimeValue defaultConnectionTimeout;
     private final TimeValue defaultReadTimeout;
     private final ByteSizeValue maxResponseSize;
+    private final CryptoService cryptoService;
 
-    public HttpClient(Settings settings, HttpAuthRegistry httpAuthRegistry, SSLService sslService) {
+    public HttpClient(Settings settings, SSLService sslService, CryptoService cryptoService) {
         super(settings);
-        this.httpAuthRegistry = httpAuthRegistry;
         this.defaultConnectionTimeout = HttpSettings.CONNECTION_TIMEOUT.get(settings);
         this.defaultReadTimeout = HttpSettings.READ_TIMEOUT.get(settings);
         this.maxResponseSize = HttpSettings.MAX_HTTP_RESPONSE_SIZE.get(settings);
         this.settingsProxy = getProxyFromSettings();
+        this.cryptoService = cryptoService;
 
         HttpClientBuilder clientBuilder = HttpClientBuilder.create();
 
@@ -139,9 +140,10 @@ public HttpResponse execute(HttpRequest request) throws IOException {
         HttpClientContext localContext = HttpClientContext.create();
         // auth
         if (request.auth() != null) {
-            ApplicableHttpAuth applicableAuth = httpAuthRegistry.createApplicable(request.auth);
             CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-            applicableAuth.apply(credentialsProvider, new AuthScope(request.host, request.port));
+            Credentials credentials = new UsernamePasswordCredentials(request.auth().username,
+                new String(request.auth().password.text(cryptoService)));
+            credentialsProvider.setCredentials(new AuthScope(request.host, request.port), credentials);
             localContext.setCredentialsProvider(credentialsProvider);
 
             // preemptive auth, no need to wait for a 401 first
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java
index 3754cc440eb64..6bcfc9e5e7d5f 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java
@@ -21,8 +21,6 @@
 import org.elasticsearch.xpack.core.watcher.support.WatcherUtils;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherXContentParser;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -50,7 +48,7 @@ public class HttpRequest implements ToXContentObject {
     @Nullable final String path;
     final Map<String, String> params;
     final Map<String, String> headers;
-    @Nullable final HttpAuth auth;
+    @Nullable final BasicAuth auth;
     @Nullable final String body;
     @Nullable final TimeValue connectionTimeout;
     @Nullable final TimeValue readTimeout;
@@ -58,7 +56,7 @@ public class HttpRequest implements ToXContentObject {
 
     public HttpRequest(String host, int port, @Nullable Scheme scheme, @Nullable HttpMethod method, @Nullable String path,
                        @Nullable Map<String, String> params, @Nullable Map<String, String> headers,
-                       @Nullable HttpAuth auth, @Nullable String body, @Nullable TimeValue connectionTimeout,
+                       @Nullable BasicAuth auth, @Nullable String body, @Nullable TimeValue connectionTimeout,
                        @Nullable TimeValue readTimeout, @Nullable HttpProxy proxy) {
         this.host = host;
         this.port = port;
@@ -102,7 +100,7 @@ public Map<String, String> headers() {
         return headers;
     }
 
-    public HttpAuth auth() {
+    public BasicAuth auth() {
         return auth;
     }
 
@@ -166,7 +164,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params toX
         }
         if (auth != null) {
             builder.startObject(Field.AUTH.getPreferredName())
-                        .field(auth.type(), auth, toXContentParams)
+                        .field(BasicAuth.TYPE, auth, toXContentParams)
                     .endObject();
         }
         if (body != null) {
@@ -234,7 +232,7 @@ public String toString() {
             sb.append("], ");
         }
         if (auth != null) {
-            sb.append("auth=[").append(auth.type()).append("], ");
+            sb.append("auth=[").append(BasicAuth.TYPE).append("], ");
         }
         sb.append("connection_timeout=[").append(connectionTimeout).append("], ");
         sb.append("read_timeout=[").append(readTimeout).append("], ");
@@ -254,14 +252,7 @@ static Builder builder() {
     }
 
     public static class Parser {
-
-        private final HttpAuthRegistry httpAuthRegistry;
-
-        public Parser(HttpAuthRegistry httpAuthRegistry) {
-            this.httpAuthRegistry = httpAuthRegistry;
-        }
-
-        public HttpRequest parse(XContentParser parser) throws IOException {
+        public static HttpRequest parse(XContentParser parser) throws IOException {
             Builder builder = new Builder();
             XContentParser.Token token;
             String currentFieldName = null;
@@ -275,7 +266,7 @@ public HttpRequest parse(XContentParser parser) throws IOException {
                         throw new ElasticsearchParseException("could not parse http request. could not parse [{}] field", currentFieldName);
                     }
                 } else if (Field.AUTH.match(currentFieldName, parser.getDeprecationHandler())) {
-                    builder.auth(httpAuthRegistry.parse(parser));
+                    builder.auth(BasicAuth.parse(parser));
                 } else if (HttpRequest.Field.CONNECTION_TIMEOUT.match(currentFieldName, parser.getDeprecationHandler())) {
                     builder.connectionTimeout(TimeValue.timeValueMillis(parser.longValue()));
                 } else if (HttpRequest.Field.CONNECTION_TIMEOUT_HUMAN.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -302,7 +293,7 @@ public HttpRequest parse(XContentParser parser) throws IOException {
                         builder.setHeaders((Map) WatcherUtils.flattenModel(parser.map()));
                     } else if (Field.PARAMS.match(currentFieldName, parser.getDeprecationHandler())) {
                         builder.setParams((Map) WatcherUtils.flattenModel(parser.map()));
-                    }  else if (Field.BODY.match(currentFieldName, parser.getDeprecationHandler())) {
+                    } else if (Field.BODY.match(currentFieldName, parser.getDeprecationHandler())) {
                         builder.body(parser.text());
                     } else {
                         throw new ElasticsearchParseException("could not parse http request. unexpected object field [{}]",
@@ -360,7 +351,7 @@ public static class Builder {
         private String path;
         private Map<String, String> params = new HashMap<>();
         private Map<String, String> headers = new HashMap<>();
-        private HttpAuth auth;
+        private BasicAuth auth;
         private String body;
         private TimeValue connectionTimeout;
         private TimeValue readTimeout;
@@ -421,7 +412,7 @@ public Builder setHeader(String key, String value) {
             return this;
         }
 
-        public Builder auth(HttpAuth auth) {
+        public Builder auth(BasicAuth auth) {
             this.auth = auth;
             return this;
         }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplate.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplate.java
index e511b30ef5934..8ca87395248b1 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplate.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplate.java
@@ -18,8 +18,6 @@
 import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.xpack.core.watcher.support.WatcherDateTimeUtils;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 
@@ -42,14 +40,14 @@ public class HttpRequestTemplate implements ToXContentObject {
     private final TextTemplate path;
     private final Map<String, TextTemplate> params;
     private final Map<String, TextTemplate> headers;
-    private final HttpAuth auth;
+    private final BasicAuth auth;
     private final TextTemplate body;
     @Nullable private final TimeValue connectionTimeout;
     @Nullable private final TimeValue readTimeout;
     @Nullable private final HttpProxy proxy;
 
     public HttpRequestTemplate(String host, int port, @Nullable Scheme scheme, @Nullable HttpMethod method, @Nullable TextTemplate path,
-                               Map<String, TextTemplate> params, Map<String, TextTemplate> headers, HttpAuth auth,
+                               Map<String, TextTemplate> params, Map<String, TextTemplate> headers, BasicAuth auth,
                                TextTemplate body, @Nullable TimeValue connectionTimeout, @Nullable TimeValue readTimeout,
                                @Nullable HttpProxy proxy) {
         this.host = host;
@@ -94,7 +92,7 @@ public Map<String, TextTemplate> headers() {
         return headers;
     }
 
-    public HttpAuth auth() {
+    public BasicAuth auth() {
         return auth;
     }
 
@@ -185,7 +183,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
         }
         if (auth != null) {
             builder.startObject(HttpRequest.Field.AUTH.getPreferredName())
-                    .field(auth.type(), auth, params)
+                    .field(BasicAuth.TYPE, auth, params)
                     .endObject();
         }
         if (body != null) {
@@ -261,14 +259,7 @@ static Builder builder() {
     }
 
     public static class Parser {
-
-        private final HttpAuthRegistry httpAuthRegistry;
-
-        public Parser(HttpAuthRegistry httpAuthRegistry) {
-            this.httpAuthRegistry = httpAuthRegistry;
-        }
-
-        public HttpRequestTemplate parse(XContentParser parser) throws IOException {
+        public static HttpRequestTemplate parse(XContentParser parser) throws IOException {
             assert parser.currentToken() == XContentParser.Token.START_OBJECT;
 
             Builder builder = new Builder();
@@ -312,8 +303,8 @@ public HttpRequestTemplate parse(XContentParser parser) throws IOException {
                     }
                 } else if (token == XContentParser.Token.START_OBJECT) {
                     if (HttpRequest.Field.AUTH.match(currentFieldName, parser.getDeprecationHandler())) {
-                        builder.auth(httpAuthRegistry.parse(parser));
-                    }  else {
+                        builder.auth(BasicAuth.parse(parser));
+                    } else {
                         throw new ElasticsearchParseException("could not parse http request template. unexpected object field [{}]",
                                 currentFieldName);
                     }
@@ -387,7 +378,7 @@ public static class Builder {
         private TextTemplate path;
         private final Map<String, TextTemplate> params = new HashMap<>();
         private final Map<String, TextTemplate> headers = new HashMap<>();
-        private HttpAuth auth;
+        private BasicAuth auth;
         private TextTemplate body;
         private TimeValue connectionTimeout;
         private TimeValue readTimeout;
@@ -444,7 +435,7 @@ public Builder putHeader(String key, TextTemplate value) {
             return this;
         }
 
-        public Builder auth(HttpAuth auth) {
+        public Builder auth(BasicAuth auth) {
             this.auth = auth;
             return this;
         }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/ApplicableHttpAuth.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/ApplicableHttpAuth.java
deleted file mode 100644
index cecc7bfda79ed..0000000000000
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/ApplicableHttpAuth.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.watcher.common.http.auth;
-
-import org.apache.http.auth.AuthScope;
-import org.apache.http.client.CredentialsProvider;
-import org.elasticsearch.common.xcontent.ToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-
-public abstract class ApplicableHttpAuth<Auth extends HttpAuth> implements ToXContentObject {
-
-    protected final Auth auth;
-
-    public ApplicableHttpAuth(Auth auth) {
-        this.auth = auth;
-    }
-
-    public final String type() {
-        return auth.type();
-    }
-
-    public abstract void apply(HttpURLConnection connection);
-
-    public abstract void apply(CredentialsProvider credsProvider, AuthScope authScope);
-
-    @Override
-    public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return auth.toXContent(builder, params);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-
-        ApplicableHttpAuth<?> that = (ApplicableHttpAuth<?>) o;
-
-        return auth.equals(that.auth);
-    }
-
-    @Override
-    public int hashCode() {
-        return auth.hashCode();
-    }
-}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuth.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuth.java
deleted file mode 100644
index 0909a33ab7312..0000000000000
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuth.java
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.watcher.common.http.auth;
-
-import org.elasticsearch.common.xcontent.ToXContentObject;
-
-public interface HttpAuth extends ToXContentObject {
-
-    String type();
-
-}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthFactory.java
deleted file mode 100644
index 7667e568b0ad3..0000000000000
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.watcher.common.http.auth;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-
-import java.io.IOException;
-
-public abstract class HttpAuthFactory<Auth extends HttpAuth, AAuth extends ApplicableHttpAuth<Auth>> {
-
-    public abstract String type();
-
-    public abstract Auth parse(XContentParser parser) throws IOException;
-
-    public abstract AAuth createApplicable(Auth auth);
-
-    public AAuth parseApplicable(XContentParser parser) throws IOException {
-        Auth auth = parse(parser);
-        return createApplicable(auth);
-    }
-
-}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthRegistry.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthRegistry.java
deleted file mode 100644
index edf584231d636..0000000000000
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/HttpAuthRegistry.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.watcher.common.http.auth;
-
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static org.elasticsearch.xpack.core.watcher.support.Exceptions.illegalArgument;
-
-public class HttpAuthRegistry {
-
-    private final Map<String, HttpAuthFactory> factories;
-
-    public HttpAuthRegistry(Map<String, HttpAuthFactory> factories) {
-        this.factories = factories;
-    }
-
-    public HttpAuth parse(XContentParser parser) throws IOException {
-        String type = null;
-        XContentParser.Token token;
-        HttpAuth auth = null;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                type = parser.currentName();
-            } else if (token == XContentParser.Token.START_OBJECT && type != null) {
-                HttpAuthFactory factory = factories.get(type);
-                if (factory == null) {
-                    throw new ElasticsearchParseException("unknown http auth type [{}]", type);
-                }
-                auth = factory.parse(parser);
-            }
-        }
-        return auth;
-    }
-
-    public <A extends HttpAuth, AA extends ApplicableHttpAuth<A>> AA createApplicable(A auth) {
-        HttpAuthFactory factory = factories.get(auth.type());
-        if (factory == null) {
-            throw illegalArgument("unknown http auth type [{}]", auth.type());
-        }
-        return (AA) factory.createApplicable(auth);
-    }
-
-}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/ApplicableBasicAuth.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/ApplicableBasicAuth.java
deleted file mode 100644
index 86639a59a1242..0000000000000
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/ApplicableBasicAuth.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.watcher.common.http.auth.basic;
-
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.elasticsearch.xpack.core.watcher.crypto.CryptoService;
-import org.elasticsearch.xpack.watcher.common.http.auth.ApplicableHttpAuth;
-
-import java.net.HttpURLConnection;
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-
-public class ApplicableBasicAuth extends ApplicableHttpAuth<BasicAuth> {
-
-    private final String basicAuth;
-    private final CryptoService cryptoService;
-
-    public ApplicableBasicAuth(BasicAuth auth, CryptoService service) {
-        super(auth);
-        basicAuth = headerValue(auth.username, auth.password.text(service));
-        this.cryptoService = service;
-    }
-
-    public static String headerValue(String username, char[] password) {
-        return "Basic " + Base64.getEncoder().encodeToString((username + ":" + new String(password)).getBytes(StandardCharsets.UTF_8));
-    }
-
-    public void apply(HttpURLConnection connection) {
-        connection.setRequestProperty("Authorization", basicAuth);
-    }
-
-    @Override
-    public void apply(CredentialsProvider credsProvider, AuthScope authScope) {
-        credsProvider.setCredentials(authScope,
-                new UsernamePasswordCredentials(auth.username, new String(auth.password.text(cryptoService))));
-    }
-
-}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuthFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuthFactory.java
deleted file mode 100644
index 838c53be7b19a..0000000000000
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuthFactory.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.watcher.common.http.auth.basic;
-
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.watcher.crypto.CryptoService;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory;
-
-import java.io.IOException;
-
-public class BasicAuthFactory extends HttpAuthFactory<BasicAuth, ApplicableBasicAuth> {
-
-    private final CryptoService cryptoService;
-
-    public BasicAuthFactory(@Nullable CryptoService cryptoService) {
-        this.cryptoService = cryptoService;
-    }
-
-    public String type() {
-        return BasicAuth.TYPE;
-    }
-
-    public BasicAuth parse(XContentParser parser) throws IOException {
-        return BasicAuth.parse(parser);
-    }
-
-    @Override
-    public ApplicableBasicAuth createApplicable(BasicAuth auth) {
-        return new ApplicableBasicAuth(auth, cryptoService);
-    }
-}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInput.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInput.java
index 3f3acdc0b3235..4a0763f7d6b69 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInput.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInput.java
@@ -70,7 +70,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static HttpInput parse(String watchId, XContentParser parser, HttpRequestTemplate.Parser requestParser) throws IOException {
+    public static HttpInput parse(String watchId, XContentParser parser) throws IOException {
         Set<String> extract = null;
         HttpRequestTemplate request = null;
         HttpContentType expectedResponseBodyType = null;
@@ -82,7 +82,7 @@ public static HttpInput parse(String watchId, XContentParser parser, HttpRequest
                 currentFieldName = parser.currentName();
             } else if (Field.REQUEST.match(currentFieldName, parser.getDeprecationHandler())) {
                 try {
-                    request = requestParser.parse(parser);
+                    request = HttpRequestTemplate.Parser.parse(parser);
                 } catch (ElasticsearchParseException pe) {
                     throw new ElasticsearchParseException("could not parse [{}] input for watch [{}]. failed to parse http request " +
                             "template", pe, TYPE, watchId);
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInputFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInputFactory.java
index 2cb26ab1d6707..7a68a77ddd21b 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInputFactory.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInputFactory.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.watcher.common.http.HttpClient;
-import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.input.InputFactory;
 
@@ -19,14 +18,11 @@ public final class HttpInputFactory extends InputFactory<HttpInput, HttpInput.Re
 
     private final HttpClient httpClient;
     private final TextTemplateEngine templateEngine;
-    private final HttpRequestTemplate.Parser requestTemplateParser;
 
-    public HttpInputFactory(Settings settings, HttpClient httpClient, TextTemplateEngine templateEngine,
-                            HttpRequestTemplate.Parser requestTemplateParser) {
+    public HttpInputFactory(Settings settings, HttpClient httpClient, TextTemplateEngine templateEngine) {
         super(Loggers.getLogger(ExecutableHttpInput.class, settings));
         this.templateEngine = templateEngine;
         this.httpClient = httpClient;
-        this.requestTemplateParser = requestTemplateParser;
     }
 
     @Override
@@ -36,7 +32,7 @@ public String type() {
 
     @Override
     public HttpInput parseInput(String watchId, XContentParser parser) throws IOException {
-        return HttpInput.parse(watchId, parser, requestTemplateParser);
+        return HttpInput.parse(watchId, parser);
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParser.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParser.java
index a9f161a3fc8a4..7c5d68a126b2c 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParser.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParser.java
@@ -34,13 +34,10 @@ public interface Fields {
 
     public static final String TYPE = "http";
     private final HttpClient httpClient;
-    private HttpRequestTemplate.Parser requestTemplateParser;
     private final TextTemplateEngine templateEngine;
 
-    public HttpEmailAttachementParser(HttpClient httpClient, HttpRequestTemplate.Parser requestTemplateParser,
-                                      TextTemplateEngine templateEngine) {
+    public HttpEmailAttachementParser(HttpClient httpClient, TextTemplateEngine templateEngine) {
         this.httpClient = httpClient;
-        this.requestTemplateParser = requestTemplateParser;
         this.templateEngine = templateEngine;
     }
 
@@ -65,7 +62,7 @@ public HttpRequestAttachment parse(String id, XContentParser parser) throws IOEx
             } else if (Fields.INLINE.match(currentFieldName, parser.getDeprecationHandler())) {
                 inline = parser.booleanValue();
             } else if (Fields.REQUEST.match(currentFieldName, parser.getDeprecationHandler())) {
-                requestTemplate = requestTemplateParser.parse(parser);
+                requestTemplate = HttpRequestTemplate.Parser.parse(parser);
             } else {
                 String msg = "Unknown field name [" + currentFieldName + "] in http request attachment configuration";
                 throw new ElasticsearchParseException(msg);
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachment.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachment.java
index b4c0208ae2a1c..f5d330f5323f6 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachment.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachment.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.watcher.common.http.HttpProxy;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -26,14 +26,14 @@ public class ReportingAttachment implements EmailAttachmentParser.EmailAttachmen
 
     private final boolean inline;
     private final String id;
-    private final HttpAuth auth;
+    private final BasicAuth auth;
     private final String url;
     private final TimeValue interval;
     private final Integer retries;
     private final HttpProxy proxy;
 
     ReportingAttachment(String id, String url, boolean inline, @Nullable TimeValue interval, @Nullable Integer retries,
-                        @Nullable HttpAuth auth, @Nullable HttpProxy proxy) {
+                        @Nullable BasicAuth auth, @Nullable HttpProxy proxy) {
         this.id = id;
         this.url = url;
         this.retries = retries;
@@ -61,7 +61,7 @@ public boolean inline() {
         return inline;
     }
 
-    public HttpAuth auth() {
+    public BasicAuth auth() {
         return auth;
     }
 
@@ -100,7 +100,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
         if (auth != null) {
             builder.startObject(AUTH.getPreferredName());
-            builder.field(auth.type(), auth, params);
+            builder.field(BasicAuth.TYPE, auth, params);
             builder.endObject();
         }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParser.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParser.java
index cb50876678374..28a8c194b579f 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParser.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParser.java
@@ -28,8 +28,7 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.notification.email.Attachment;
@@ -69,15 +68,12 @@ public class ReportingAttachmentParser implements EmailAttachmentParser<Reportin
     private final int retries;
     private HttpClient httpClient;
     private final TextTemplateEngine templateEngine;
-    private HttpAuthRegistry authRegistry;
 
-    public ReportingAttachmentParser(Settings settings, HttpClient httpClient,
-                                     TextTemplateEngine templateEngine, HttpAuthRegistry authRegistry) {
+    public ReportingAttachmentParser(Settings settings, HttpClient httpClient, TextTemplateEngine templateEngine) {
         this.interval = INTERVAL_SETTING.get(settings);
         this.retries = RETRIES_SETTING.get(settings);
         this.httpClient = httpClient;
         this.templateEngine = templateEngine;
-        this.authRegistry = authRegistry;
         this.logger = Loggers.getLogger(getClass());
     }
 
@@ -89,7 +85,7 @@ public String type() {
     @Override
     public ReportingAttachment parse(String id, XContentParser parser) throws IOException {
         Builder builder = new Builder(id);
-        PARSER.parse(parser, builder, new AuthParseContext(authRegistry));
+        PARSER.parse(parser, builder, new AuthParseContext());
         return builder.build();
     }
 
@@ -222,15 +218,9 @@ private String extractIdFromJson(String watchId, String attachmentId, BytesRefer
      */
     private static class AuthParseContext {
 
-        private final HttpAuthRegistry authRegistry;
-
-        AuthParseContext(HttpAuthRegistry authRegistry) {
-            this.authRegistry = authRegistry;
-        }
-
-        HttpAuth parseAuth(XContentParser parser) {
+        BasicAuth parseAuth(XContentParser parser) {
             try {
-                return authRegistry.parse(parser);
+                return BasicAuth.parse(parser);
             } catch (IOException e) {
                 throw new UncheckedIOException(e);
             }
@@ -273,7 +263,7 @@ static class Builder {
         private String url;
         private TimeValue interval;
         private Integer retries;
-        private HttpAuth auth;
+        private BasicAuth auth;
         private HttpProxy proxy;
 
         Builder(String id) {
@@ -301,7 +291,7 @@ Builder inline(boolean inline) {
             return this;
         }
 
-        Builder auth(HttpAuth auth) {
+        Builder auth(BasicAuth auth) {
             this.auth = auth;
             return this;
         }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatMessage.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatMessage.java
index ae08bb85e9dfd..10eaba535edf1 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatMessage.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatMessage.java
@@ -13,14 +13,12 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
-import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
 
 public class HipChatMessage implements ToXContentObject {
@@ -181,26 +179,6 @@ public int hashCode() {
             return Objects.hash(body, rooms, users, from, format, color, notify);
         }
 
-        public HipChatMessage render(TextTemplateEngine engine, Map<String, Object> model) {
-            String body = engine.render(this.body, model);
-            String[] rooms = null;
-            if (this.rooms != null) {
-                rooms = new String[this.rooms.length];
-                for (int i = 0; i < this.rooms.length; i++) {
-                    rooms[i] = engine.render(this.rooms[i], model);
-                }
-            }
-            String[] users = null;
-            if (this.users != null) {
-                users = new String[this.users.length];
-                for (int i = 0; i < this.users.length; i++) {
-                    users[i] = engine.render(this.users[i], model);
-                }
-            }
-            Color color = this.color == null ? null : Color.resolve(engine.render(this.color, model), null);
-            return new HipChatMessage(body, rooms, users, from, format, color, notify);
-        }
-
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccount.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccount.java
index 5efe09c575924..4221b405d299c 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccount.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccount.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
 import org.elasticsearch.xpack.watcher.common.http.Scheme;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 
 import java.io.IOException;
 import java.io.InputStream;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEvent.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEvent.java
index 0fb1a52d28633..c44fbf36e0b18 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEvent.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEvent.java
@@ -24,22 +24,22 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * Official documentation for this can be found at
  *
- * https://developer.pagerduty.com/documentation/howto/manually-trigger-an-incident/
- * https://developer.pagerduty.com/documentation/integration/events/trigger
- * https://developer.pagerduty.com/documentation/integration/events/acknowledge
- * https://developer.pagerduty.com/documentation/integration/events/resolve
+ * https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2
  */
 public class IncidentEvent implements ToXContentObject {
 
     static final String HOST = "events.pagerduty.com";
-    static final String PATH = "/generic/2010-04-15/create_event.json";
+    static final String PATH = "/v2/enqueue";
+    static final String ACCEPT_HEADER = "application/vnd.pagerduty+json;version=2";
 
     final String description;
     @Nullable final HttpProxy proxy;
@@ -93,46 +93,81 @@ public int hashCode() {
         return result;
     }
 
-    public HttpRequest createRequest(final String serviceKey, final Payload payload) throws IOException {
+    HttpRequest createRequest(final String serviceKey, final Payload payload, final String watchId) throws IOException {
         return HttpRequest.builder(HOST, -1)
                 .method(HttpMethod.POST)
                 .scheme(Scheme.HTTPS)
                 .path(PATH)
                 .proxy(proxy)
-                .jsonBody(new ToXContent() {
-                    @Override
-                    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-                        builder.field(Fields.SERVICE_KEY.getPreferredName(), serviceKey);
-                        builder.field(Fields.EVENT_TYPE.getPreferredName(), eventType);
-                        builder.field(Fields.DESCRIPTION.getPreferredName(), description);
-                        if (incidentKey != null) {
-                            builder.field(Fields.INCIDENT_KEY.getPreferredName(), incidentKey);
-                        }
-                        if (client != null) {
-                            builder.field(Fields.CLIENT.getPreferredName(), client);
-                        }
-                        if (clientUrl != null) {
-                            builder.field(Fields.CLIENT_URL.getPreferredName(), clientUrl);
-                        }
-                        if (attachPayload) {
-                            builder.startObject(Fields.DETAILS.getPreferredName());
-                            builder.field(Fields.PAYLOAD.getPreferredName());
-                            payload.toXContent(builder, params);
-                            builder.endObject();
-                        }
-                        if (contexts != null && contexts.length > 0) {
-                            builder.startArray(Fields.CONTEXTS.getPreferredName());
-                            for (IncidentEventContext context : contexts) {
-                                context.toXContent(builder, params);
-                            }
-                            builder.endArray();
-                        }
-                        return builder;
-                    }
-                })
+                .setHeader("Accept", ACCEPT_HEADER)
+                .jsonBody((b, p) -> buildAPIXContent(b, p, serviceKey, payload, watchId))
                 .build();
     }
 
+    XContentBuilder buildAPIXContent(XContentBuilder builder, Params params, String serviceKey,
+                                            Payload payload, String watchId) throws IOException {
+        builder.field(Fields.ROUTING_KEY.getPreferredName(), serviceKey);
+        builder.field(Fields.EVENT_ACTION.getPreferredName(), eventType);
+        if (incidentKey != null) {
+            builder.field(Fields.DEDUP_KEY.getPreferredName(), incidentKey);
+        }
+
+        builder.startObject(Fields.PAYLOAD.getPreferredName());
+        {
+            builder.field(Fields.SUMMARY.getPreferredName(), description);
+
+            if (attachPayload && payload != null) {
+                builder.startObject(Fields.CUSTOM_DETAILS.getPreferredName());
+                {
+                    builder.field(Fields.PAYLOAD.getPreferredName(), payload, params);
+                }
+                builder.endObject();
+            }
+
+            if (watchId != null) {
+                builder.field(Fields.SOURCE.getPreferredName(), watchId);
+            } else {
+                builder.field(Fields.SOURCE.getPreferredName(), "watcher");
+            }
+            // TODO externalize this into something user editable
+            builder.field(Fields.SEVERITY.getPreferredName(), "critical");
+        }
+        builder.endObject();
+
+        if (client != null) {
+            builder.field(Fields.CLIENT.getPreferredName(), client);
+        }
+        if (clientUrl != null) {
+            builder.field(Fields.CLIENT_URL.getPreferredName(), clientUrl);
+        }
+
+        if (contexts != null && contexts.length > 0) {
+            toXContentV2Contexts(builder, params, contexts);
+        }
+
+        return builder;
+    }
+
+    /**
+     * Turns the V1 API contexts into 2 distinct lists, images and links. The V2 API has separated these out into 2 top level fields.
+     */
+    private void toXContentV2Contexts(XContentBuilder builder, ToXContent.Params params,
+                                      IncidentEventContext[] contexts) throws IOException {
+        // contexts can be either links or images, and the v2 api needs them separate
+        Map<IncidentEventContext.Type, List<IncidentEventContext>> groups = Arrays.stream(contexts)
+            .collect(Collectors.groupingBy(iec -> iec.type));
+
+        List<IncidentEventContext> links = groups.getOrDefault(IncidentEventContext.Type.LINK, Collections.emptyList());
+        if (links.isEmpty() == false) {
+            builder.array(Fields.LINKS.getPreferredName(), links.toArray());
+        }
+
+        List<IncidentEventContext> images = groups.getOrDefault(IncidentEventContext.Type.IMAGE, Collections.emptyList());
+        if (images.isEmpty() == false) {
+            builder.array(Fields.IMAGES.getPreferredName(), images.toArray());
+        }
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject();
@@ -445,8 +480,15 @@ interface Fields {
         // we need to keep this for BWC
         ParseField CONTEXT_DEPRECATED = new ParseField("context");
 
-        ParseField SERVICE_KEY = new ParseField("service_key");
         ParseField PAYLOAD = new ParseField("payload");
-        ParseField DETAILS = new ParseField("details");
+        ParseField ROUTING_KEY = new ParseField("routing_key");
+        ParseField EVENT_ACTION = new ParseField("event_action");
+        ParseField DEDUP_KEY = new ParseField("dedup_key");
+        ParseField SUMMARY = new ParseField("summary");
+        ParseField SOURCE = new ParseField("source");
+        ParseField SEVERITY = new ParseField("severity");
+        ParseField LINKS = new ParseField("links");
+        ParseField IMAGES = new ParseField("images");
+        ParseField CUSTOM_DETAILS = new ParseField("custom_details");
     }
 }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventContext.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventContext.java
index d43829346b626..cd9924ae9dca7 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventContext.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventContext.java
@@ -92,6 +92,85 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder.endObject();
     }
 
+    public static IncidentEventContext parse(XContentParser parser) throws IOException {
+        Type type = null;
+        String href = null;
+        String text = null;
+        String src = null;
+        String alt = null;
+
+        String currentFieldName = null;
+        XContentParser.Token token;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (Strings.hasLength(currentFieldName)) {
+                if (XField.TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
+                    try {
+                        type = Type.valueOf(parser.text().toUpperCase(Locale.ROOT));
+                    } catch (IllegalArgumentException e) {
+                        String msg = "could not parse trigger incident event context. unknown context type [{}]";
+                        throw new ElasticsearchParseException(msg, parser.text());
+                    }
+                } else {
+                    if (XField.HREF.match(currentFieldName, parser.getDeprecationHandler())) {
+                        href = parser.text();
+                    } else if (XField.TEXT.match(currentFieldName, parser.getDeprecationHandler())) {
+                        text = parser.text();
+                    } else if (XField.SRC.match(currentFieldName, parser.getDeprecationHandler())) {
+                        src = parser.text();
+                    } else if (XField.ALT.match(currentFieldName, parser.getDeprecationHandler())) {
+                        alt = parser.text();
+                    } else {
+                        String msg = "could not parse trigger incident event context. unknown field [{}]";
+                        throw new ElasticsearchParseException(msg, currentFieldName);
+                    }
+                }
+            }
+        }
+
+        return createAndValidateTemplate(type, href, src, alt, text);
+    }
+
+    private static IncidentEventContext createAndValidateTemplate(Type type, String href, String src, String alt,
+                                                      String text) {
+        if (type == null) {
+            throw new ElasticsearchParseException("could not parse trigger incident event context. missing required field [{}]",
+                XField.TYPE.getPreferredName());
+        }
+
+        switch (type) {
+            case LINK:
+                if (href == null) {
+                    throw new ElasticsearchParseException("could not parse trigger incident event context. missing required field " +
+                        "[{}] for [{}] context", XField.HREF.getPreferredName(), Type.LINK.name().toLowerCase(Locale.ROOT));
+                }
+                if (src != null) {
+                    throw new ElasticsearchParseException("could not parse trigger incident event context. unexpected field [{}] for " +
+                        "[{}] context", XField.SRC.getPreferredName(), Type.LINK.name().toLowerCase(Locale.ROOT));
+                }
+                if (alt != null) {
+                    throw new ElasticsearchParseException("could not parse trigger incident event context. unexpected field [{}] for " +
+                        "[{}] context", XField.ALT.getPreferredName(), Type.LINK.name().toLowerCase(Locale.ROOT));
+                }
+                return link(href, text);
+            case IMAGE:
+                if (src == null) {
+                    throw new ElasticsearchParseException("could not parse trigger incident event context. missing required field " +
+                        "[{}] for [{}] context", XField.SRC.getPreferredName(), Type.IMAGE.name().toLowerCase(Locale.ROOT));
+                }
+                if (text != null) {
+                    throw new ElasticsearchParseException("could not parse trigger incident event context. unexpected field [{}] for " +
+                        "[{}] context", XField.TEXT.getPreferredName(), Type.IMAGE.name().toLowerCase(Locale.ROOT));
+                }
+                return image(src, href, alt);
+            default:
+                throw new ElasticsearchParseException("could not parse trigger incident event context. unknown context type [{}]",
+                    type);
+        }
+    }
+
+
     public static class Template implements ToXContentObject {
 
         final Type type;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccount.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccount.java
index 5cf1a77f9711a..b2498a749d7b2 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccount.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccount.java
@@ -48,8 +48,8 @@ public IncidentEventDefaults getDefaults() {
         return eventDefaults;
     }
 
-    public SentEvent send(IncidentEvent event, Payload payload) throws IOException {
-        HttpRequest request = event.createRequest(serviceKey, payload);
+    public SentEvent send(IncidentEvent event, Payload payload, String watchId) throws IOException {
+        HttpRequest request = event.createRequest(serviceKey, payload, watchId);
         HttpResponse response = httpClient.execute(request);
         return SentEvent.responded(event, request, response);
     }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
index e41d52c1954fe..28eb884bbe872 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/delete/TransportDeleteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/delete/TransportDeleteWatchAction.java
index d7ff25b623f50..9cc94820554dc 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/delete/TransportDeleteWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/delete/TransportDeleteWatchAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 
 import java.util.function.Supplier;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailActionTests.java
index 83b48cb9f4f0a..d073d3e4dd1e3 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailActionTests.java
@@ -27,8 +27,6 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.notification.email.Attachment;
@@ -79,7 +77,6 @@
 
 public class EmailActionTests extends ESTestCase {
 
-    private HttpAuthRegistry registry = new HttpAuthRegistry(singletonMap("basic", new BasicAuthFactory(null)));
     private HttpClient httpClient = mock(HttpClient.class);
     private EmailAttachmentsParser emailAttachmentParser;
 
@@ -87,7 +84,7 @@ public class EmailActionTests extends ESTestCase {
     public void addEmailAttachmentParsers() {
         Map<String, EmailAttachmentParser> emailAttachmentParsers = new HashMap<>();
         emailAttachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient,
-                new HttpRequestTemplate.Parser(registry), new MockTextTemplateEngine()));
+            new MockTextTemplateEngine()));
         emailAttachmentParsers.put(DataAttachmentParser.TYPE, new DataAttachmentParser());
         emailAttachmentParser = new EmailAttachmentsParser(emailAttachmentParsers);
     }
@@ -511,10 +508,8 @@ public void testThatOneFailedEmailAttachmentResultsInActionFailure() throws Exce
                 .thenReturn(new HttpResponse(403));
 
         // setup email attachment parsers
-        HttpRequestTemplate.Parser httpRequestTemplateParser = new HttpRequestTemplate.Parser(registry);
         Map<String, EmailAttachmentParser> attachmentParsers = new HashMap<>();
-        attachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, httpRequestTemplateParser,
-                engine));
+        attachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, engine));
         EmailAttachmentsParser emailAttachmentsParser = new EmailAttachmentsParser(attachmentParsers);
 
         XContentBuilder builder = jsonBuilder().startObject()
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java
index cb434e62df5f9..aa03bf4545e5b 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java
@@ -17,8 +17,7 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpProxy;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.notification.jira.JiraAccount;
@@ -93,11 +92,9 @@ public void testProxy() throws Exception {
         assertThat(request.port(), is(port));
         assertThat(request.path(), is(JiraAccount.DEFAULT_PATH));
 
-        HttpAuth httpAuth = request.auth();
-        assertThat(httpAuth.type(), is("basic"));
+        BasicAuth httpAuth = request.auth();
 
-        BasicAuth basicAuth = (BasicAuth) httpAuth;
-        assertThat(basicAuth.getUsername(), is(user));
+        assertThat(httpAuth.getUsername(), is(user));
     }
 
     public void testExecutionWithNoDefaults() throws Exception {
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionTests.java
index 6f57ccd82d930..07a55c628ec1c 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionTests.java
@@ -111,7 +111,7 @@ public void testExecute() throws Exception {
         when(response.status()).thenReturn(200);
         HttpRequest request = mock(HttpRequest.class);
         SentEvent sentEvent = SentEvent.responded(event, request, response);
-        when(account.send(event, payload)).thenReturn(sentEvent);
+        when(account.send(event, payload, wid.watchId())).thenReturn(sentEvent);
         when(service.getAccount(accountName)).thenReturn(account);
 
         Action.Result result = executable.execute("_id", ctx, payload);
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java
index 09ca57c1708f7..755e3dffe5c90 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java
@@ -28,8 +28,6 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.execution.TriggeredExecutionContext;
@@ -47,7 +45,6 @@
 import java.io.IOException;
 import java.util.Map;
 
-import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.CoreMatchers.instanceOf;
@@ -73,7 +70,6 @@ public class WebhookActionTests extends ESTestCase {
     private static final String TEST_PATH_STRING = "/testPath";
 
     private TextTemplateEngine templateEngine;
-    private HttpAuthRegistry authRegistry;
     private TextTemplate testBody;
     private TextTemplate testPath;
 
@@ -82,7 +78,6 @@ public void init() throws Exception {
         templateEngine = new MockTextTemplateEngine();
         testBody = new TextTemplate(TEST_BODY_STRING);
         testPath = new TextTemplate(TEST_PATH_STRING);
-        authRegistry = new HttpAuthRegistry(singletonMap("basic", new BasicAuthFactory(null)));
     }
 
     public void testExecute() throws Exception {
@@ -213,14 +208,14 @@ public void testParserFailure() throws Exception {
     }
 
     private WebhookActionFactory webhookFactory(HttpClient client) {
-        return new WebhookActionFactory(Settings.EMPTY, client, new HttpRequestTemplate.Parser(authRegistry), templateEngine);
+        return new WebhookActionFactory(Settings.EMPTY, client, templateEngine);
     }
 
     public void testThatSelectingProxyWorks() throws Exception {
         Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
 
-        try (HttpClient httpClient = new HttpClient(Settings.EMPTY, authRegistry,
-            new SSLService(environment.settings(), environment)); MockWebServer proxyServer = new MockWebServer()) {
+        try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null);
+             MockWebServer proxyServer = new MockWebServer()) {
             proxyServer.start();
             proxyServer.enqueue(new MockResponse().setResponseCode(200).setBody("fullProxiedContent"));
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java
index a69445386d027..0ba733ea4c5de 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpMethod;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.common.http.Scheme;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java
index 4fd162495f491..d26d07a8fbdb6 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.xpack.watcher.actions.ActionBuilders;
 import org.elasticsearch.xpack.watcher.common.http.HttpMethod;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
index 03dcd7947155e..882dc5389a493 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
@@ -27,9 +27,6 @@
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.ssl.TestsSSLService;
 import org.elasticsearch.xpack.core.ssl.VerificationMode;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
 import org.junit.After;
 import org.junit.Before;
 
@@ -48,7 +45,6 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -64,14 +60,12 @@ public class HttpClientTests extends ESTestCase {
 
     private MockWebServer webServer = new MockWebServer();
     private HttpClient httpClient;
-    private HttpAuthRegistry authRegistry;
     private Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
 
     @Before
     public void init() throws Exception {
-        authRegistry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(null)));
         webServer.start();
-        httpClient = new HttpClient(Settings.EMPTY, authRegistry, new SSLService(environment.settings(), environment));
+        httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null);
     }
 
     @After
@@ -185,7 +179,7 @@ public void testHttps() throws Exception {
                 .setSecureSettings(secureSettings)
                 .build();
         }
-        try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) {
+        try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) {
             secureSettings = new MockSecureSettings();
             // We can't use the client created above for the server since it is only a truststore
             secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
@@ -226,7 +220,7 @@ public void testHttpsDisableHostnameVerification() throws Exception {
             }
             settings = builder.build();
         }
-        try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) {
+        try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) {
             MockSecureSettings secureSettings = new MockSecureSettings();
             // We can't use the client created above for the server since it only defines a truststore
             secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode-no-subjaltname");
@@ -253,7 +247,7 @@ public void testHttpsClientAuth() throws Exception {
             .build();
 
         TestsSSLService sslService = new TestsSSLService(settings, environment);
-        try (HttpClient client = new HttpClient(settings, authRegistry, sslService)) {
+        try (HttpClient client = new HttpClient(settings, sslService, null)) {
             testSslMockWebserver(client, sslService.sslContext(), true);
         }
     }
@@ -301,7 +295,7 @@ public void testHttpResponseWithAnyStatusCodeCanReturnBody() throws Exception {
 
     @Network
     public void testHttpsWithoutTruststore() throws Exception {
-        try (HttpClient client = new HttpClient(Settings.EMPTY, authRegistry, new SSLService(Settings.EMPTY, environment))) {
+        try (HttpClient client = new HttpClient(Settings.EMPTY, new SSLService(Settings.EMPTY, environment), null)) {
             // Known server with a valid cert from a commercial CA
             HttpRequest.Builder request = HttpRequest.builder("www.elastic.co", 443).scheme(Scheme.HTTPS);
             HttpResponse response = client.execute(request.build());
@@ -325,7 +319,7 @@ public void testThatProxyCanBeConfigured() throws Exception {
                     .method(HttpMethod.GET)
                     .path("/");
 
-            try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) {
+            try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) {
                 HttpResponse response = client.execute(requestBuilder.build());
                 assertThat(response.status(), equalTo(200));
                 assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent"));
@@ -406,7 +400,7 @@ public void testProxyCanHaveDifferentSchemeThanRequest() throws Exception {
                     .scheme(Scheme.HTTP)
                     .path("/");
 
-            try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) {
+            try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) {
                 HttpResponse response = client.execute(requestBuilder.build());
                 assertThat(response.status(), equalTo(200));
                 assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent"));
@@ -434,7 +428,7 @@ public void testThatProxyCanBeOverriddenByRequest() throws Exception {
                     .proxy(new HttpProxy("localhost", proxyServer.getPort(), Scheme.HTTP))
                     .path("/");
 
-            try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) {
+            try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) {
                 HttpResponse response = client.execute(requestBuilder.build());
                 assertThat(response.status(), equalTo(200));
                 assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent"));
@@ -455,7 +449,7 @@ public void testThatProxyConfigurationRequiresHostAndPort() {
         }
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> new HttpClient(settings.build(), authRegistry, new SSLService(settings.build(), environment)));
+                () -> new HttpClient(settings.build(), new SSLService(settings.build(), environment), null));
         assertThat(e.getMessage(),
                 containsString("HTTP proxy requires both settings: [xpack.http.proxy.host] and [xpack.http.proxy.port]"));
     }
@@ -554,7 +548,7 @@ public void testMaxHttpResponseSize() throws Exception {
 
         HttpRequest.Builder requestBuilder = HttpRequest.builder("localhost", webServer.getPort()).method(HttpMethod.GET).path("/");
 
-        try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(environment.settings(), environment))) {
+        try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null)) {
             IOException e = expectThrows(IOException.class, () -> client.execute(requestBuilder.build()));
             assertThat(e.getMessage(), startsWith("Maximum limit of"));
         }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java
index 8ac2bef16e8d8..21efe5b2b94ea 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java
@@ -13,11 +13,9 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.Network;
 import org.elasticsearch.xpack.core.ssl.SSLService;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
 
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.lessThan;
-import static org.mockito.Mockito.mock;
 
 public class HttpConnectionTimeoutTests extends ESTestCase {
     // setting an unroutable IP to simulate a connection timeout
@@ -26,8 +24,7 @@ public class HttpConnectionTimeoutTests extends ESTestCase {
     @Network
     public void testDefaultTimeout() throws Exception {
         Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
-        HttpClient httpClient = new HttpClient(Settings.EMPTY, mock(HttpAuthRegistry.class),
-                new SSLService(environment.settings(), environment));
+        HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null);
 
         HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345)
                 .method(HttpMethod.POST)
@@ -52,8 +49,7 @@ public void testDefaultTimeout() throws Exception {
     public void testDefaultTimeoutCustom() throws Exception {
         Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
         HttpClient httpClient = new HttpClient(Settings.builder()
-                .put("xpack.http.default_connection_timeout", "5s").build()
-                , mock(HttpAuthRegistry.class), new SSLService(environment.settings(), environment));
+                .put("xpack.http.default_connection_timeout", "5s").build(), new SSLService(environment.settings(), environment), null);
 
         HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345)
                 .method(HttpMethod.POST)
@@ -78,8 +74,7 @@ public void testDefaultTimeoutCustom() throws Exception {
     public void testTimeoutCustomPerRequest() throws Exception {
         Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
         HttpClient httpClient = new HttpClient(Settings.builder()
-                .put("xpack.http.default_connection_timeout", "10s").build()
-                , mock(HttpAuthRegistry.class), new SSLService(environment.settings(), environment));
+                .put("xpack.http.default_connection_timeout", "10s").build(), new SSLService(environment.settings(), environment), null);
 
         HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345)
                 .connectionTimeout(TimeValue.timeValueSeconds(5))
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java
index fa5a53f4e1da0..bc328dc586e9e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.ssl.SSLService;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
 import org.junit.After;
 import org.junit.Before;
 
@@ -21,7 +20,6 @@
 
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.lessThan;
-import static org.mockito.Mockito.mock;
 
 public class HttpReadTimeoutTests extends ESTestCase {
 
@@ -45,8 +43,7 @@ public void testDefaultTimeout() throws Exception {
                 .path("/")
                 .build();
 
-        try (HttpClient httpClient = new HttpClient(Settings.EMPTY, mock(HttpAuthRegistry.class),
-            new SSLService(environment.settings(), environment))) {
+        try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null)) {
             long start = System.nanoTime();
 
             expectThrows(SocketTimeoutException.class, () ->  httpClient.execute(request));
@@ -68,8 +65,7 @@ public void testDefaultTimeoutCustom() throws Exception {
                 .build();
 
         try (HttpClient httpClient = new HttpClient(Settings.builder()
-            .put("xpack.http.default_read_timeout", "3s").build()
-            , mock(HttpAuthRegistry.class), new SSLService(environment.settings(), environment))) {
+            .put("xpack.http.default_read_timeout", "3s").build(), new SSLService(environment.settings(), environment), null)) {
 
             long start = System.nanoTime();
             expectThrows(SocketTimeoutException.class, () ->  httpClient.execute(request));
@@ -92,8 +88,7 @@ public void testTimeoutCustomPerRequest() throws Exception {
                 .build();
 
         try (HttpClient httpClient = new HttpClient(Settings.builder()
-            .put("xpack.http.default_read_timeout", "10s").build()
-            , mock(HttpAuthRegistry.class), new SSLService(environment.settings(), environment))) {
+            .put("xpack.http.default_read_timeout", "10s").build(), new SSLService(environment.settings(), environment), null)) {
 
             long start = System.nanoTime();
             expectThrows(SocketTimeoutException.class, () ->  httpClient.execute(request));
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplateTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplateTests.java
index 1f57c812ebbe4..dcbc19f4d5070 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplateTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTemplateTests.java
@@ -14,22 +14,17 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.test.MockTextTemplateEngine;
 
 import java.util.Collections;
 
 import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
-import static org.mockito.Mockito.mock;
 
 public class HttpRequestTemplateTests extends ESTestCase {
 
@@ -87,8 +82,7 @@ public void testProxyParsing() throws Exception {
         XContentParser xContentParser = createParser(xContentBuilder);
         xContentParser.nextToken();
 
-        HttpRequestTemplate.Parser parser = new HttpRequestTemplate.Parser(mock(HttpAuthRegistry.class));
-        HttpRequestTemplate parsedTemplate = parser.parse(xContentParser);
+        HttpRequestTemplate parsedTemplate = HttpRequestTemplate.Parser.parse(xContentParser);
         assertThat(parsedTemplate.proxy().getPort(), is(proxyPort));
         assertThat(parsedTemplate.proxy().getHost(), is(proxyHost));
     }
@@ -134,14 +128,10 @@ public void testParseSelfGenerated() throws Exception {
 
         HttpRequestTemplate template = builder.build();
 
-        HttpAuthRegistry registry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE,
-                new BasicAuthFactory(null)));
-        HttpRequestTemplate.Parser parser = new HttpRequestTemplate.Parser(registry);
-
         XContentBuilder xContentBuilder = template.toXContent(jsonBuilder(), WatcherParams.builder().hideSecrets(false).build());
         XContentParser xContentParser = createParser(xContentBuilder);
         xContentParser.nextToken();
-        HttpRequestTemplate parsed = parser.parse(xContentParser);
+        HttpRequestTemplate parsed = HttpRequestTemplate.Parser.parse(xContentParser);
 
         assertEquals(template, parsed);
     }
@@ -194,13 +184,12 @@ private void assertThatManualBuilderEqualsParsingFromUrl(String url, HttpRequest
         XContentParser urlContentParser = createParser(urlContentBuilder);
         urlContentParser.nextToken();
 
-        HttpRequestTemplate.Parser parser = new HttpRequestTemplate.Parser(mock(HttpAuthRegistry.class));
-        HttpRequestTemplate urlParsedTemplate = parser.parse(urlContentParser);
+        HttpRequestTemplate urlParsedTemplate = HttpRequestTemplate.Parser.parse(urlContentParser);
 
         XContentBuilder xContentBuilder = builder.build().toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS);
         XContentParser xContentParser = createParser(xContentBuilder);
         xContentParser.nextToken();
-        HttpRequestTemplate parsedTemplate = parser.parse(xContentParser);
+        HttpRequestTemplate parsedTemplate = HttpRequestTemplate.Parser.parse(xContentParser);
 
         assertThat(parsedTemplate, is(urlParsedTemplate));
     }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java
index 223735cad54e4..0d1541577a58b 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java
@@ -13,11 +13,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherXContentParser;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
 
-import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.cborBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
@@ -25,7 +21,6 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
-import static org.mockito.Mockito.mock;
 
 public class HttpRequestTests extends ESTestCase {
 
@@ -129,14 +124,12 @@ public void testXContentSerialization() throws Exception {
         try (XContentBuilder xContentBuilder = randomFrom(jsonBuilder(), smileBuilder(), yamlBuilder(), cborBuilder())) {
             httpRequest.toXContent(xContentBuilder, WatcherParams.builder().hideSecrets(false).build());
 
-            HttpAuthRegistry registry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(null)));
-            HttpRequest.Parser httpRequestParser = new HttpRequest.Parser(registry);
-    
+
             try (XContentParser parser = createParser(xContentBuilder)) {
                 assertNull(parser.currentToken());
                 parser.nextToken();
-    
-                HttpRequest parsedRequest = httpRequestParser.parse(parser);
+
+                HttpRequest parsedRequest = HttpRequest.Parser.parse(parser);
                 assertEquals(httpRequest, parsedRequest);
             }
         }
@@ -161,14 +154,13 @@ private void assertThatManualBuilderEqualsParsingFromUrl(String url, HttpRequest
         XContentParser urlContentParser = createParser(urlContentBuilder);
         urlContentParser.nextToken();
 
-        HttpRequest.Parser parser = new HttpRequest.Parser(mock(HttpAuthRegistry.class));
-        HttpRequest urlParsedRequest = parser.parse(urlContentParser);
+        HttpRequest urlParsedRequest = HttpRequest.Parser.parse(urlContentParser);
 
         WatcherParams params = WatcherParams.builder().hideSecrets(false).build();
         XContentBuilder xContentBuilder = builder.build().toXContent(jsonBuilder(), params);
         XContentParser xContentParser = createParser(xContentBuilder);
         xContentParser.nextToken();
-        HttpRequest parsedRequest = parser.parse(xContentParser);
+        HttpRequest parsedRequest = HttpRequest.Parser.parse(xContentParser);
 
         assertThat(parsedRequest, is(urlParsedRequest));
     }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
index d3f46d3d452f7..bb593bcb67ae2 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
@@ -33,7 +33,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.core.watcher.actions.Action;
 import org.elasticsearch.xpack.core.watcher.actions.ActionStatus;
 import org.elasticsearch.xpack.core.watcher.actions.ActionWrapper;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputTests.java
index cc19cef7b4768..2e1e7858f16d9 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputTests.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.xpack.core.watcher.input.Input;
 import org.elasticsearch.xpack.core.watcher.watch.Payload;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.condition.ScriptCondition;
 import org.elasticsearch.xpack.watcher.input.InputFactory;
 import org.elasticsearch.xpack.watcher.input.InputRegistry;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java
index dd13fcb14a252..7d81ff2203d23 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java
@@ -28,10 +28,7 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
 import org.elasticsearch.xpack.watcher.common.http.Scheme;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.input.InputBuilders;
@@ -73,8 +70,7 @@ public class HttpInputTests extends ESTestCase {
     public void init() throws Exception {
         httpClient = mock(HttpClient.class);
         templateEngine = mock(TextTemplateEngine.class);
-        HttpAuthRegistry registry = new HttpAuthRegistry(singletonMap("basic", new BasicAuthFactory(null)));
-        httpParser = new HttpInputFactory(Settings.EMPTY, httpClient, templateEngine, new HttpRequestTemplate.Parser(registry));
+        httpParser = new HttpInputFactory(Settings.EMPTY, httpClient, templateEngine);
     }
 
     public void testExecute() throws Exception {
@@ -158,7 +154,7 @@ public void testParser() throws Exception {
                 randomBoolean() ? new MapBuilder<String, TextTemplate>().put("a", new TextTemplate("b")).map() : null;
         Map<String, TextTemplate> headers =
                 randomBoolean() ? new MapBuilder<String, TextTemplate>().put("c", new TextTemplate("d")).map() : null;
-        HttpAuth auth = randomBoolean() ? new BasicAuth("username", "password".toCharArray()) : null;
+        BasicAuth auth = randomBoolean() ? new BasicAuth("username", "password".toCharArray()) : null;
         HttpRequestTemplate.Builder requestBuilder = HttpRequestTemplate.builder(host, port)
                 .scheme(scheme)
                 .method(httpMethod)
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParserTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParserTests.java
index ef71a1157437b..2b0e632b0cac7 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParserTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParserTests.java
@@ -19,9 +19,6 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
 import org.elasticsearch.xpack.watcher.test.MockTextTemplateEngine;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -34,7 +31,6 @@
 import java.util.Map;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.mockExecutionContextBuilder;
 import static org.hamcrest.Matchers.hasSize;
@@ -45,20 +41,17 @@
 
 public class HttpEmailAttachementParserTests extends ESTestCase {
 
-    private HttpRequestTemplate.Parser httpRequestTemplateParser;
     private HttpClient httpClient;
     private EmailAttachmentsParser emailAttachmentsParser;
     private Map<String, EmailAttachmentParser> attachmentParsers;
 
     @Before
     public void init() throws Exception {
-        HttpAuthRegistry authRegistry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(null)));
-        httpRequestTemplateParser = new HttpRequestTemplate.Parser(authRegistry);
         httpClient = mock(HttpClient.class);
 
         attachmentParsers = new HashMap<>();
         attachmentParsers.put(HttpEmailAttachementParser.TYPE,
-                new HttpEmailAttachementParser(httpClient, httpRequestTemplateParser, new MockTextTemplateEngine()));
+                new HttpEmailAttachementParser(httpClient, new MockTextTemplateEngine()));
         emailAttachmentsParser = new EmailAttachmentsParser(attachmentParsers);
     }
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java
index e8678f5c40e91..97907bbef8a59 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java
@@ -8,7 +8,6 @@
 import com.fasterxml.jackson.core.io.JsonEOFException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -24,11 +23,7 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpProxy;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.notification.email.Attachment;
@@ -69,19 +64,13 @@ public class ReportingAttachmentParserTests extends ESTestCase {
     private Map<String, EmailAttachmentParser> attachmentParsers = new HashMap<>();
     private EmailAttachmentsParser emailAttachmentsParser;
     private ReportingAttachmentParser reportingAttachmentParser;
-    private HttpAuthRegistry authRegistry;
     private MockTextTemplateEngine templateEngine = new MockTextTemplateEngine();
     private String dashboardUrl = "http://www.example.org/ovb/api/reporting/generate/dashboard/My-Dashboard";
 
     @Before
     public void init() throws Exception {
         httpClient = mock(HttpClient.class);
-
-        Map<String, HttpAuthFactory> factories = MapBuilder.<String, HttpAuthFactory>newMapBuilder()
-                .put("basic", new BasicAuthFactory(null))
-                .immutableMap();
-        authRegistry = new HttpAuthRegistry(factories);
-        reportingAttachmentParser = new ReportingAttachmentParser(Settings.EMPTY, httpClient, templateEngine, authRegistry);
+        reportingAttachmentParser = new ReportingAttachmentParser(Settings.EMPTY, httpClient, templateEngine);
 
         attachmentParsers.put(ReportingAttachmentParser.TYPE, reportingAttachmentParser);
         emailAttachmentsParser = new EmailAttachmentsParser(attachmentParsers);
@@ -113,7 +102,7 @@ public void testSerializationWorks() throws Exception {
             builder.field("inline", true);
         }
 
-        HttpAuth auth = null;
+        BasicAuth auth = null;
         boolean withAuth = randomBoolean();
         boolean isPasswordEncrypted = randomBoolean();
         if (withAuth) {
@@ -318,9 +307,8 @@ public void testWithBasicAuth() throws Exception {
         assertThat(allRequests, hasSize(3));
         for (HttpRequest request : allRequests) {
             assertThat(request.auth(), is(notNullValue()));
-            assertThat(request.auth().type(), is("basic"));
             assertThat(request.auth(), instanceOf(BasicAuth.class));
-            BasicAuth basicAuth = (BasicAuth) request.auth();
+            BasicAuth basicAuth = request.auth();
             assertThat(basicAuth.getUsername(), is("foo"));
         }
     }
@@ -351,7 +339,7 @@ public void testPollingDefaultCanBeOverriddenBySettings() throws Exception {
                 .put(ReportingAttachmentParser.RETRIES_SETTING.getKey(), retries)
                 .build();
 
-        reportingAttachmentParser = new ReportingAttachmentParser(settings, httpClient, templateEngine, authRegistry);
+        reportingAttachmentParser = new ReportingAttachmentParser(settings, httpClient, templateEngine);
         expectThrows(ElasticsearchException.class, () ->
                 reportingAttachmentParser.toAttachment(createWatchExecutionContext(), Payload.EMPTY, attachment));
 
@@ -374,7 +362,7 @@ public String render(TextTemplate textTemplate, Map<String, Object> model) {
         ReportingAttachment attachment = new ReportingAttachment("foo", "http://www.example.org/REPLACEME", randomBoolean(),
                 TimeValue.timeValueMillis(1), 10, new BasicAuth("foo", "bar".toCharArray()), null);
         reportingAttachmentParser = new ReportingAttachmentParser(Settings.EMPTY, httpClient,
-                replaceHttpWithHttpsTemplateEngine, authRegistry);
+                replaceHttpWithHttpsTemplateEngine);
         reportingAttachmentParser.toAttachment(createWatchExecutionContext(), Payload.EMPTY, attachment);
 
         ArgumentCaptor<HttpRequest> requestArgumentCaptor = ArgumentCaptor.forClass(HttpRequest.class);
@@ -391,7 +379,7 @@ public void testRetrySettingCannotBeNegative() throws Exception {
 
         Settings invalidSettings = Settings.builder().put("xpack.notification.reporting.retries", -10).build();
         e = expectThrows(IllegalArgumentException.class,
-                () -> new ReportingAttachmentParser(invalidSettings, httpClient, templateEngine, authRegistry));
+                () -> new ReportingAttachmentParser(invalidSettings, httpClient, templateEngine));
         assertThat(e.getMessage(), is("Failed to parse value [-10] for setting [xpack.notification.reporting.retries] must be >= 0"));
     }
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatAccountsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatAccountsTests.java
index b793bf9202382..795f5aaacc7c9 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatAccountsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatAccountsTests.java
@@ -12,12 +12,9 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpProxy;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
-import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
-import org.elasticsearch.xpack.watcher.test.MockTextTemplateEngine;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
 
-import java.util.HashMap;
 import java.util.HashSet;
 
 import static org.hamcrest.Matchers.is;
@@ -40,11 +37,7 @@ public void testProxy() throws Exception {
                 new HashSet<>(HipChatService.getSettings())));
         HipChatAccount account = service.getAccount("account1");
 
-        HipChatMessage.Template template = new HipChatMessage.Template.Builder(new TextTemplate("foo"))
-                .addRooms(new TextTemplate("room"))
-                .setFrom("from")
-                .build();
-        HipChatMessage hipChatMessage = template.render(new MockTextTemplateEngine(), new HashMap<>());
+        HipChatMessage hipChatMessage = new HipChatMessage("body", new String[]{"rooms"}, null, "from", null, null, null);
 
         ArgumentCaptor<HttpRequest> argumentCaptor = ArgumentCaptor.forClass(HttpRequest.class);
         when(httpClient.execute(argumentCaptor.capture())).thenReturn(new HttpResponse(200));
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraIssueTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraIssueTests.java
index 8e615d76050a7..e3c4d79f6d0e5 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraIssueTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraIssueTests.java
@@ -14,14 +14,11 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpMethod;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 
 import java.util.HashMap;
 import java.util.Map;
 
-import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.cborBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
@@ -43,19 +40,19 @@ public void testToXContent() throws Exception {
 
             Map<String, Object> parsedFields = null;
             Map<String, Object> parsedResult = null;
-    
+
             HttpRequest parsedRequest = null;
             HttpResponse parsedResponse = null;
             String parsedAccount = null;
             String parsedReason = null;
-    
+
             try (XContentParser parser = createParser(builder)) {
                 assertNull(parser.currentToken());
                 parser.nextToken();
-    
+
                 XContentParser.Token token = parser.currentToken();
                 assertThat(token, is(XContentParser.Token.START_OBJECT));
-    
+
                 String currentFieldName = null;
                 while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                     if (token == XContentParser.Token.FIELD_NAME) {
@@ -65,9 +62,7 @@ public void testToXContent() throws Exception {
                     } else if ("result".equals(currentFieldName)) {
                         parsedResult = parser.map();
                     } else if ("request".equals(currentFieldName)) {
-                        HttpAuthRegistry registry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(null)));
-                        HttpRequest.Parser httpRequestParser = new HttpRequest.Parser(registry);
-                        parsedRequest = httpRequestParser.parse(parser);
+                        parsedRequest = HttpRequest.Parser.parse(parser);
                     } else if ("response".equals(currentFieldName)) {
                         parsedResponse = HttpResponse.parse(parser);
                     } else if ("fields".equals(currentFieldName)) {
@@ -79,7 +74,7 @@ public void testToXContent() throws Exception {
                     }
                 }
             }
-            
+
             assertThat(parsedAccount, equalTo(issue.getAccount()));
             assertThat(parsedFields, equalTo(issue.getFields()));
             if (issue.successful()) {
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventTests.java
new file mode 100644
index 0000000000000..3638d5f85d929
--- /dev/null
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEventTests.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.watcher.notification.pagerduty;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
+import org.elasticsearch.xpack.core.watcher.watch.Payload;
+import org.elasticsearch.xpack.watcher.common.http.HttpProxy;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+
+public class IncidentEventTests extends ESTestCase {
+
+    public void testPagerDutyXContent() throws IOException {
+
+        String serviceKey = randomAlphaOfLength(3);
+        boolean attachPayload = randomBoolean();
+        Payload payload = null;
+        if (attachPayload) {
+            payload = new Payload.Simple(Collections.singletonMap(randomAlphaOfLength(3), randomAlphaOfLength(3)));
+        }
+        String watchId = randomAlphaOfLength(3);
+        String description = randomAlphaOfLength(3);
+        String eventType = randomAlphaOfLength(3);
+        String incidentKey = rarely() ? null : randomAlphaOfLength(3);
+        String client = rarely() ? null : randomAlphaOfLength(3);
+        String clientUrl = rarely() ? null : randomAlphaOfLength(3);
+        String account = rarely() ? null : randomAlphaOfLength(3);
+
+        IncidentEventContext[] contexts = null;
+        List<IncidentEventContext> links = new ArrayList<>();
+        List<IncidentEventContext> images = new ArrayList<>();
+
+        if (randomBoolean()) {
+            int numContexts = randomIntBetween(0, 3);
+            contexts = new IncidentEventContext[numContexts];
+            for (int i = 0; i < numContexts; i++) {
+                if (randomBoolean()) {
+                    contexts[i] = IncidentEventContext.link("href", "text");
+                    links.add(contexts[i]);
+                } else {
+                    contexts[i] = IncidentEventContext.image("src", "href", "alt");
+                    images.add(contexts[i]);
+                }
+            }
+        }
+
+        HttpProxy proxy = rarely() ? null : HttpProxy.NO_PROXY;
+
+        IncidentEvent event = new IncidentEvent(description, eventType, incidentKey, client, clientUrl, account,
+            attachPayload, contexts, proxy);
+
+        XContentBuilder jsonBuilder = jsonBuilder();
+        jsonBuilder.startObject(); // since its a snippet
+        event.buildAPIXContent(jsonBuilder, ToXContent.EMPTY_PARAMS, serviceKey, payload, watchId);
+        jsonBuilder.endObject();
+        XContentParser parser = createParser(jsonBuilder);
+        parser.nextToken();
+
+        ObjectPath objectPath = ObjectPath.createFromXContent(jsonBuilder.contentType().xContent(), BytesReference.bytes(jsonBuilder));
+
+        String actualServiceKey = objectPath.evaluate(IncidentEvent.Fields.ROUTING_KEY.getPreferredName());
+        String actualWatchId = objectPath.evaluate(IncidentEvent.Fields.PAYLOAD.getPreferredName()
+            + "." + IncidentEvent.Fields.SOURCE.getPreferredName());
+        if (actualWatchId == null) {
+            actualWatchId = "watcher"; // hardcoded if the SOURCE is null
+        }
+        String actualDescription = objectPath.evaluate(IncidentEvent.Fields.PAYLOAD.getPreferredName()
+            + "." + IncidentEvent.Fields.SUMMARY.getPreferredName());
+        String actualEventType = objectPath.evaluate(IncidentEvent.Fields.EVENT_ACTION.getPreferredName());
+        String actualIncidentKey = objectPath.evaluate(IncidentEvent.Fields.DEDUP_KEY.getPreferredName());
+        String actualClient = objectPath.evaluate(IncidentEvent.Fields.CLIENT.getPreferredName());
+        String actualClientUrl = objectPath.evaluate(IncidentEvent.Fields.CLIENT_URL.getPreferredName());
+        String actualSeverity = objectPath.evaluate(IncidentEvent.Fields.PAYLOAD.getPreferredName()
+            + "." + IncidentEvent.Fields.SEVERITY.getPreferredName());
+        Map<String, Object> payloadDetails = objectPath.evaluate("payload.custom_details.payload");
+        Payload actualPayload = null;
+
+        if (payloadDetails != null) {
+            actualPayload = new Payload.Simple(payloadDetails);
+        }
+
+        List<IncidentEventContext> actualLinks = new ArrayList<>();
+        List<Map<String, String>> linkMap = (List<Map<String, String>>) objectPath.evaluate(IncidentEvent.Fields.LINKS.getPreferredName());
+        if (linkMap != null) {
+            for (Map<String, String> iecValue : linkMap) {
+                actualLinks.add(IncidentEventContext.link(iecValue.get("href"), iecValue.get("text")));
+            }
+        }
+
+        List<IncidentEventContext> actualImages = new ArrayList<>();
+        List<Map<String, String>> imgMap = (List<Map<String, String>>) objectPath.evaluate(IncidentEvent.Fields.IMAGES.getPreferredName());
+        if (imgMap != null) {
+            for (Map<String, String> iecValue : imgMap) {
+                actualImages.add(IncidentEventContext.image(iecValue.get("src"), iecValue.get("href"), iecValue.get("alt")));
+            }
+        }
+
+        // assert the actuals were the same as expected
+        assertThat(serviceKey, equalTo(actualServiceKey));
+        assertThat(eventType, equalTo(actualEventType));
+        assertThat(incidentKey, equalTo(actualIncidentKey));
+        assertThat(description, equalTo(actualDescription));
+        assertThat(watchId, equalTo(actualWatchId));
+        assertThat("critical", equalTo(actualSeverity));
+        assertThat(client, equalTo(actualClient));
+        assertThat(clientUrl, equalTo(actualClientUrl));
+        assertThat(links, equalTo(actualLinks));
+        assertThat(images, equalTo(actualImages));
+        assertThat(payload, equalTo(actualPayload));
+    }
+}
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccountsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccountsTests.java
index d70badc4bec22..1e88c69614270 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccountsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyAccountsTests.java
@@ -24,6 +24,7 @@
 
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -48,7 +49,7 @@ public void testProxy() throws Exception {
 
         HttpProxy proxy = new HttpProxy("localhost", 8080);
         IncidentEvent event = new IncidentEvent("foo", null, null, null, null, account.getName(), true, null, proxy);
-        account.send(event, Payload.EMPTY);
+        account.send(event, Payload.EMPTY, null);
 
         HttpRequest request = argumentCaptor.getValue();
         assertThat(request.proxy(), is(proxy));
@@ -72,11 +73,13 @@ public void testContextIsSentCorrect() throws Exception {
                         "https://www.elastic.co/products/x-pack/alerting", "X-Pack-Alerting website link with log")
         };
         IncidentEvent event = new IncidentEvent("foo", null, null, null, null, account.getName(), true, contexts, HttpProxy.NO_PROXY);
-        account.send(event, Payload.EMPTY);
+        account.send(event, Payload.EMPTY, null);
 
         HttpRequest request = argumentCaptor.getValue();
         ObjectPath source = ObjectPath.createFromXContent(JsonXContent.jsonXContent, new BytesArray(request.body()));
-        assertThat(source.evaluate("contexts"), notNullValue());
+        assertThat(source.evaluate("contexts"), nullValue());
+        assertThat(source.evaluate("links"), notNullValue());
+        assertThat(source.evaluate("images"), notNullValue());
     }
 
     private void addAccountSettings(String name, Settings.Builder builder) {
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
index 18f151b4f6afb..c07e6ba3db113 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.condition.CompareCondition;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java
index 9b7050dc30d79..0de8fc1ee4bc0 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java
@@ -22,9 +22,8 @@
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.ApplicableBasicAuth;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTriggerEvent;
@@ -32,6 +31,8 @@
 import org.junit.After;
 import org.junit.Before;
 
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -154,7 +155,7 @@ public void testHttpInput() throws Exception {
 
         assertThat(webServer.requests(), hasSize(1));
         assertThat(webServer.requests().get(0).getHeader("Authorization"),
-                is(ApplicableBasicAuth.headerValue(USERNAME, PASSWORD.toCharArray())));
+                is(headerValue(USERNAME, PASSWORD.toCharArray())));
 
         // now trigger the by the scheduler and make sure that the password is also correctly transmitted
         webServer.enqueue(new MockResponse().setResponseCode(200).setBody(
@@ -162,10 +163,9 @@ public void testHttpInput() throws Exception {
         timeWarp().trigger("_id");
         assertThat(webServer.requests(), hasSize(2));
         assertThat(webServer.requests().get(1).getHeader("Authorization"),
-            is(ApplicableBasicAuth.headerValue(USERNAME, PASSWORD.toCharArray())));
+            is(headerValue(USERNAME, PASSWORD.toCharArray())));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30094")
     public void testWebhookAction() throws Exception {
         WatcherClient watcherClient = watcherClient();
         watcherClient.preparePutWatch("_id")
@@ -252,6 +252,10 @@ public void testWebhookAction() throws Exception {
 
         assertThat(webServer.requests(), hasSize(1));
         assertThat(webServer.requests().get(0).getHeader("Authorization"),
-                is(ApplicableBasicAuth.headerValue(USERNAME, PASSWORD.toCharArray())));
+                is(headerValue(USERNAME, PASSWORD.toCharArray())));
+    }
+
+    private String headerValue(String username, char[] password) {
+        return "Basic " + Base64.getEncoder().encodeToString((username + ":" + new String(password)).getBytes(StandardCharsets.UTF_8));
     }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java
index 64b913f96a0df..62985cad1f144 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
index 76f71b9c95ed4..2f502cb95aac3 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
index df96a802166e2..8029b1c7a49cd 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
@@ -61,8 +61,6 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpClient;
 import org.elasticsearch.xpack.watcher.common.http.HttpMethod;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
-import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.condition.AlwaysConditionTests;
@@ -160,7 +158,6 @@ public class WatchTests extends ESTestCase {
     private EmailService emailService;
     private TextTemplateEngine templateEngine;
     private HtmlSanitizer htmlSanitizer;
-    private HttpAuthRegistry authRegistry;
     private XPackLicenseState licenseState;
     private Logger logger;
     private Settings settings = Settings.EMPTY;
@@ -175,7 +172,6 @@ public void init() throws Exception {
         templateEngine = mock(TextTemplateEngine.class);
         htmlSanitizer = mock(HtmlSanitizer.class);
         licenseState = mock(XPackLicenseState.class);
-        authRegistry = new HttpAuthRegistry(singletonMap("basic", new BasicAuthFactory(null)));
         logger = Loggers.getLogger(WatchTests.class);
         searchTemplateService = mock(WatcherSearchTemplateService.class);
     }
@@ -623,8 +619,7 @@ private ActionRegistry registry(List<ActionWrapper> actions, ConditionRegistry c
                     parsers.put(IndexAction.TYPE, new IndexActionFactory(settings, client));
                     break;
                 case WebhookAction.TYPE:
-                    parsers.put(WebhookAction.TYPE, new WebhookActionFactory(settings,  httpClient,
-                            new HttpRequestTemplate.Parser(authRegistry), templateEngine));
+                    parsers.put(WebhookAction.TYPE, new WebhookActionFactory(settings, httpClient, templateEngine));
                     break;
                 case LoggingAction.TYPE:
                     parsers.put(LoggingAction.TYPE, new LoggingActionFactory(settings, new MockTextTemplateEngine()));
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/DeleteLicenseRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/DeleteLicenseRequest.java
new file mode 100644
index 0000000000000..abcc4a68eb88a
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/DeleteLicenseRequest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.license;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+
+
+public class DeleteLicenseRequest extends AcknowledgedRequest<DeleteLicenseRequest> {
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WritePipelineResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/DeleteLicenseResponse.java
similarity index 70%
rename from server/src/main/java/org/elasticsearch/action/ingest/WritePipelineResponse.java
rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/DeleteLicenseResponse.java
index 293a62b66f26e..596af7f2f90e4 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/WritePipelineResponse.java
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/DeleteLicenseResponse.java
@@ -16,23 +16,21 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
-package org.elasticsearch.action.ingest;
+package org.elasticsearch.protocol.xpack.license;
 
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentParser;
 
-public class WritePipelineResponse extends AcknowledgedResponse implements ToXContentObject {
+public class DeleteLicenseResponse extends AcknowledgedResponse {
 
-    WritePipelineResponse() {
+    public DeleteLicenseResponse() {
     }
 
-    public WritePipelineResponse(boolean acknowledged) {
+    public DeleteLicenseResponse(boolean acknowledged) {
         super(acknowledged);
     }
 
-    public static WritePipelineResponse fromXContent(XContentParser parser) {
-        return new WritePipelineResponse(parseAcknowledged(parser));
+    public static DeleteLicenseResponse fromXContent(XContentParser parser) {
+        return new DeleteLicenseResponse(parseAcknowledged(parser));
     }
 }
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/GetLicenseRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/GetLicenseRequest.java
new file mode 100644
index 0000000000000..c669d3d33770c
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/GetLicenseRequest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.license;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+
+
+public class GetLicenseRequest extends MasterNodeReadRequest<GetLicenseRequest> {
+
+    public GetLicenseRequest() {
+    }
+
+    public GetLicenseRequest(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/GetLicenseResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/GetLicenseResponse.java
new file mode 100644
index 0000000000000..7232e185a7ef5
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/GetLicenseResponse.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.license;
+
+import org.elasticsearch.action.ActionResponse;
+
+public class GetLicenseResponse extends ActionResponse {
+
+    private String license;
+
+    GetLicenseResponse() {
+    }
+
+    public GetLicenseResponse(String license) {
+        this.license = license;
+    }
+
+    public String getLicenseDefinition() {
+        return license;
+    }
+
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoRequest.java
new file mode 100644
index 0000000000000..ae26bc4de8d0e
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoRequest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.migration;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+public class IndexUpgradeInfoRequest extends MasterNodeReadRequest<IndexUpgradeInfoRequest> implements IndicesRequest.Replaceable {
+
+    private String[] indices = Strings.EMPTY_ARRAY;
+    private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true);
+
+    public IndexUpgradeInfoRequest(String... indices) {
+        indices(indices);
+    }
+
+    public IndexUpgradeInfoRequest(StreamInput in) throws IOException {
+        super(in);
+        indices = in.readStringArray();
+        indicesOptions = IndicesOptions.readIndicesOptions(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStringArray(indices);
+        indicesOptions.writeIndicesOptions(out);
+    }
+
+    @Override
+    public String[] indices() {
+        return indices;
+    }
+
+    @Override
+    public IndexUpgradeInfoRequest indices(String... indices) {
+        this.indices = Objects.requireNonNull(indices, "indices cannot be null");
+        return this;
+    }
+
+    @Override
+    public IndicesOptions indicesOptions() {
+        return indicesOptions;
+    }
+
+    public void indicesOptions(IndicesOptions indicesOptions) {
+        this.indicesOptions = indicesOptions;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        IndexUpgradeInfoRequest request = (IndexUpgradeInfoRequest) o;
+        return Arrays.equals(indices, request.indices) &&
+                Objects.equals(indicesOptions.toString(), request.indicesOptions.toString());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(Arrays.hashCode(indices), indicesOptions.toString());
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoResponse.java
new file mode 100644
index 0000000000000..4c1208f960ebd
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoResponse.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.migration;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class IndexUpgradeInfoResponse extends ActionResponse implements ToXContentObject {
+
+    private static final ParseField INDICES = new ParseField("indices");
+    private static final ParseField ACTION_REQUIRED = new ParseField("action_required");
+
+    private static final ConstructingObjectParser<IndexUpgradeInfoResponse, String> PARSER =
+        new ConstructingObjectParser<>("IndexUpgradeInfoResponse",
+            true,
+            (a, c) -> {
+            @SuppressWarnings("unchecked")
+                Map<String, Object> map = (Map<String, Object>)a[0];
+                Map<String, UpgradeActionRequired> actionsRequired = map.entrySet().stream()
+                    .filter(e -> {
+                        if (e.getValue() instanceof Map == false) {
+                            return false;
+                        }
+                        @SuppressWarnings("unchecked")
+                        Map<String, Object> value =(Map<String, Object>)e.getValue();
+                        return value.containsKey(ACTION_REQUIRED.getPreferredName());
+                    })
+                    .collect(Collectors.toMap(
+                        Map.Entry::getKey,
+                        e -> {
+                            @SuppressWarnings("unchecked")
+                            Map<String, Object> value = (Map<String, Object>) e.getValue();
+                            return UpgradeActionRequired.fromString((String)value.get(ACTION_REQUIRED.getPreferredName()));
+                        }
+                ));
+                return new IndexUpgradeInfoResponse(actionsRequired);
+            });
+
+    static {
+        PARSER.declareObject(constructorArg(), (p, c) -> p.map(), INDICES);
+    }
+
+
+    private Map<String, UpgradeActionRequired> actions;
+
+    public IndexUpgradeInfoResponse() {
+
+    }
+
+    public IndexUpgradeInfoResponse(Map<String, UpgradeActionRequired> actions) {
+        this.actions = actions;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        actions = in.readMap(StreamInput::readString, UpgradeActionRequired::readFromStream);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeMap(actions, StreamOutput::writeString, (out1, value) -> value.writeTo(out1));
+    }
+
+    public Map<String, UpgradeActionRequired> getActions() {
+        return actions;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.startObject(INDICES.getPreferredName());
+            for (Map.Entry<String, UpgradeActionRequired> entry : actions.entrySet()) {
+                builder.startObject(entry.getKey());
+                {
+                    builder.field(ACTION_REQUIRED.getPreferredName(), entry.getValue().toString());
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        IndexUpgradeInfoResponse response = (IndexUpgradeInfoResponse) o;
+        return Objects.equals(actions, response.actions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(actions);
+    }
+
+    public static IndexUpgradeInfoResponse fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/UpgradeActionRequired.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/UpgradeActionRequired.java
similarity index 59%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/UpgradeActionRequired.java
rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/UpgradeActionRequired.java
index 1bc4d92f33d90..c87e37be7a55a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/UpgradeActionRequired.java
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/UpgradeActionRequired.java
@@ -1,9 +1,22 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
-package org.elasticsearch.xpack.core.upgrade;
+package org.elasticsearch.protocol.xpack.migration;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/package-info.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/package-info.java
new file mode 100644
index 0000000000000..12dc8eebc1773
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/migration/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Request and Response objects for the default distribution's Migration
+ * APIs.
+ */
+package org.elasticsearch.protocol.xpack.migration;
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/PutJobRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/PutJobRequest.java
new file mode 100644
index 0000000000000..2cdf1993fccd3
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/PutJobRequest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class PutJobRequest extends ActionRequest implements ToXContentObject {
+
+    private final Job job;
+
+    public PutJobRequest(Job job) {
+        this.job = job;
+    }
+
+    public Job getJob() {
+        return job;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return job.toXContent(builder, params);
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) {
+            return true;
+        }
+
+        if (object == null || getClass() != object.getClass()) {
+            return false;
+        }
+
+        PutJobRequest request = (PutJobRequest) object;
+        return Objects.equals(job, request.job);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(job);
+    }
+
+    @Override
+    public final String toString() {
+        return Strings.toString(this);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/PutJobResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/PutJobResponse.java
new file mode 100644
index 0000000000000..b37bd35d6b17f
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/PutJobResponse.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml;
+
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class PutJobResponse implements ToXContentObject {
+
+    private Job job;
+
+    public static PutJobResponse fromXContent(XContentParser parser) throws IOException {
+        return new PutJobResponse(Job.PARSER.parse(parser, null).build());
+    }
+
+    public PutJobResponse(Job job) {
+        this.job = job;
+    }
+
+    public PutJobResponse() {
+    }
+
+    public Job getResponse() {
+        return job;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        job.toXContent(builder, params);
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) {
+            return true;
+        }
+        if (object == null || getClass() != object.getClass()) {
+            return false;
+        }
+        PutJobResponse response = (PutJobResponse) object;
+        return Objects.equals(job, response.job);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(job);
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/ChunkingConfig.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/ChunkingConfig.java
new file mode 100644
index 0000000000000..0b9d9f1204614
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/ChunkingConfig.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.datafeed;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * The description of how searches should be chunked.
+ */
+public class ChunkingConfig implements ToXContentObject {
+
+    public static final ParseField MODE_FIELD = new ParseField("mode");
+    public static final ParseField TIME_SPAN_FIELD = new ParseField("time_span");
+
+    public static final ConstructingObjectParser<ChunkingConfig, Void> PARSER = new ConstructingObjectParser<>(
+        "chunking_config", true, a -> new ChunkingConfig((Mode) a[0], (TimeValue) a[1]));
+
+    static {
+        PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return Mode.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, MODE_FIELD, ValueType.STRING);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return TimeValue.parseTimeValue(p.text(), TIME_SPAN_FIELD.getPreferredName());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, TIME_SPAN_FIELD, ValueType.STRING);
+
+    }
+
+    private final Mode mode;
+    private final TimeValue timeSpan;
+
+
+    ChunkingConfig(Mode mode, @Nullable TimeValue timeSpan) {
+        this.mode = Objects.requireNonNull(mode, MODE_FIELD.getPreferredName());
+        this.timeSpan = timeSpan;
+    }
+
+    @Nullable
+    public TimeValue getTimeSpan() {
+        return timeSpan;
+    }
+
+    Mode getMode() {
+        return mode;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(MODE_FIELD.getPreferredName(), mode);
+        if (timeSpan != null) {
+            builder.field(TIME_SPAN_FIELD.getPreferredName(), timeSpan.getStringRep());
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(mode, timeSpan);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        ChunkingConfig other = (ChunkingConfig) obj;
+        return Objects.equals(this.mode, other.mode) &&
+            Objects.equals(this.timeSpan, other.timeSpan);
+    }
+
+    public static ChunkingConfig newAuto() {
+        return new ChunkingConfig(Mode.AUTO, null);
+    }
+
+    public static ChunkingConfig newOff() {
+        return new ChunkingConfig(Mode.OFF, null);
+    }
+
+    public static ChunkingConfig newManual(TimeValue timeSpan) {
+        return new ChunkingConfig(Mode.MANUAL, timeSpan);
+    }
+
+    public enum Mode {
+        AUTO, MANUAL, OFF;
+
+        public static Mode fromString(String value) {
+            return Mode.valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedConfig.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedConfig.java
new file mode 100644
index 0000000000000..929d4dacb90fa
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedConfig.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.datafeed;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Datafeed configuration options pojo. Describes where to proactively pull input
+ * data from.
+ * <p>
+ * If a value has not been set it will be <code>null</code>. Object wrappers are
+ * used around integral types and booleans so they can take <code>null</code>
+ * values.
+ */
+public class DatafeedConfig implements ToXContentObject {
+
+    public static final int DEFAULT_SCROLL_SIZE = 1000;
+
+    public static final ParseField ID = new ParseField("datafeed_id");
+    public static final ParseField QUERY_DELAY = new ParseField("query_delay");
+    public static final ParseField FREQUENCY = new ParseField("frequency");
+    public static final ParseField INDEXES = new ParseField("indexes");
+    public static final ParseField INDICES = new ParseField("indices");
+    public static final ParseField TYPES = new ParseField("types");
+    public static final ParseField QUERY = new ParseField("query");
+    public static final ParseField SCROLL_SIZE = new ParseField("scroll_size");
+    public static final ParseField AGGREGATIONS = new ParseField("aggregations");
+    public static final ParseField AGGS = new ParseField("aggs");
+    public static final ParseField SCRIPT_FIELDS = new ParseField("script_fields");
+    public static final ParseField CHUNKING_CONFIG = new ParseField("chunking_config");
+
+    public static final ConstructingObjectParser<Builder, Void> PARSER = new ConstructingObjectParser<>(
+        "datafeed_config", true, a -> new Builder((String)a[0], (String)a[1]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), ID);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
+
+        PARSER.declareStringArray(Builder::setIndices, INDEXES);
+        PARSER.declareStringArray(Builder::setIndices, INDICES);
+        PARSER.declareStringArray(Builder::setTypes, TYPES);
+        PARSER.declareString((builder, val) ->
+            builder.setQueryDelay(TimeValue.parseTimeValue(val, QUERY_DELAY.getPreferredName())), QUERY_DELAY);
+        PARSER.declareString((builder, val) ->
+            builder.setFrequency(TimeValue.parseTimeValue(val, FREQUENCY.getPreferredName())), FREQUENCY);
+        PARSER.declareObject(Builder::setQuery, (p, c) -> AbstractQueryBuilder.parseInnerQueryBuilder(p), QUERY);
+        PARSER.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p), AGGREGATIONS);
+        PARSER.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p), AGGS);
+        PARSER.declareObject(Builder::setScriptFields, (p, c) -> {
+            List<SearchSourceBuilder.ScriptField> parsedScriptFields = new ArrayList<>();
+            while (p.nextToken() != XContentParser.Token.END_OBJECT) {
+                parsedScriptFields.add(new SearchSourceBuilder.ScriptField(p));
+            }
+            return parsedScriptFields;
+        }, SCRIPT_FIELDS);
+        PARSER.declareInt(Builder::setScrollSize, SCROLL_SIZE);
+        PARSER.declareObject(Builder::setChunkingConfig, ChunkingConfig.PARSER, CHUNKING_CONFIG);
+    }
+
+    private final String id;
+    private final String jobId;
+
+    /**
+     * The delay before starting to query a period of time
+     */
+    private final TimeValue queryDelay;
+
+    /**
+     * The frequency with which queries are executed
+     */
+    private final TimeValue frequency;
+
+    private final List<String> indices;
+    private final List<String> types;
+    private final QueryBuilder query;
+    private final AggregatorFactories.Builder aggregations;
+    private final List<SearchSourceBuilder.ScriptField> scriptFields;
+    private final Integer scrollSize;
+    private final ChunkingConfig chunkingConfig;
+
+    private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List<String> indices, List<String> types,
+                           QueryBuilder query, AggregatorFactories.Builder aggregations, List<SearchSourceBuilder.ScriptField> scriptFields,
+                           Integer scrollSize, ChunkingConfig chunkingConfig) {
+        this.id = id;
+        this.jobId = jobId;
+        this.queryDelay = queryDelay;
+        this.frequency = frequency;
+        this.indices = indices == null ? null : Collections.unmodifiableList(indices);
+        this.types = types == null ? null : Collections.unmodifiableList(types);
+        this.query = query;
+        this.aggregations = aggregations;
+        this.scriptFields = scriptFields == null ? null : Collections.unmodifiableList(scriptFields);
+        this.scrollSize = scrollSize;
+        this.chunkingConfig = chunkingConfig;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public TimeValue getQueryDelay() {
+        return queryDelay;
+    }
+
+    public TimeValue getFrequency() {
+        return frequency;
+    }
+
+    public List<String> getIndices() {
+        return indices;
+    }
+
+    public List<String> getTypes() {
+        return types;
+    }
+
+    public Integer getScrollSize() {
+        return scrollSize;
+    }
+
+    public QueryBuilder getQuery() {
+        return query;
+    }
+
+    public AggregatorFactories.Builder getAggregations() {
+        return aggregations;
+    }
+
+    public List<SearchSourceBuilder.ScriptField> getScriptFields() {
+        return scriptFields == null ? Collections.emptyList() : scriptFields;
+    }
+
+    public ChunkingConfig getChunkingConfig() {
+        return chunkingConfig;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ID.getPreferredName(), id);
+        builder.field(Job.ID.getPreferredName(), jobId);
+        if (queryDelay != null) {
+            builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep());
+        }
+        if (frequency != null) {
+            builder.field(FREQUENCY.getPreferredName(), frequency.getStringRep());
+        }
+        builder.field(INDICES.getPreferredName(), indices);
+        builder.field(TYPES.getPreferredName(), types);
+        builder.field(QUERY.getPreferredName(), query);
+        if (aggregations != null) {
+            builder.field(AGGREGATIONS.getPreferredName(), aggregations);
+        }
+        if (scriptFields != null) {
+            builder.startObject(SCRIPT_FIELDS.getPreferredName());
+            for (SearchSourceBuilder.ScriptField scriptField : scriptFields) {
+                scriptField.toXContent(builder, params);
+            }
+            builder.endObject();
+        }
+        builder.field(SCROLL_SIZE.getPreferredName(), scrollSize);
+        if (chunkingConfig != null) {
+            builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig);
+        }
+
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * The lists of indices and types are compared for equality but they are not
+     * sorted first so this test could fail simply because the indices and types
+     * lists are in different orders.
+     */
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        DatafeedConfig that = (DatafeedConfig) other;
+
+        return Objects.equals(this.id, that.id)
+            && Objects.equals(this.jobId, that.jobId)
+            && Objects.equals(this.frequency, that.frequency)
+            && Objects.equals(this.queryDelay, that.queryDelay)
+            && Objects.equals(this.indices, that.indices)
+            && Objects.equals(this.types, that.types)
+            && Objects.equals(this.query, that.query)
+            && Objects.equals(this.scrollSize, that.scrollSize)
+            && Objects.equals(this.aggregations, that.aggregations)
+            && Objects.equals(this.scriptFields, that.scriptFields)
+            && Objects.equals(this.chunkingConfig, that.chunkingConfig);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, jobId, frequency, queryDelay, indices, types, query, scrollSize, aggregations, scriptFields,
+            chunkingConfig);
+    }
+
+    public static class Builder {
+
+        private String id;
+        private String jobId;
+        private TimeValue queryDelay;
+        private TimeValue frequency;
+        private List<String> indices = Collections.emptyList();
+        private List<String> types = Collections.emptyList();
+        private QueryBuilder query = QueryBuilders.matchAllQuery();
+        private AggregatorFactories.Builder aggregations;
+        private List<SearchSourceBuilder.ScriptField> scriptFields;
+        private Integer scrollSize = DEFAULT_SCROLL_SIZE;
+        private ChunkingConfig chunkingConfig;
+
+        public Builder(String id, String jobId) {
+            this.id = Objects.requireNonNull(id, ID.getPreferredName());
+            this.jobId = Objects.requireNonNull(jobId, Job.ID.getPreferredName());
+        }
+
+        public Builder(DatafeedConfig config) {
+            this.id = config.id;
+            this.jobId = config.jobId;
+            this.queryDelay = config.queryDelay;
+            this.frequency = config.frequency;
+            this.indices = config.indices;
+            this.types = config.types;
+            this.query = config.query;
+            this.aggregations = config.aggregations;
+            this.scriptFields = config.scriptFields;
+            this.scrollSize = config.scrollSize;
+            this.chunkingConfig = config.chunkingConfig;
+        }
+
+        public Builder setIndices(List<String> indices) {
+            this.indices = Objects.requireNonNull(indices, INDICES.getPreferredName());
+            return this;
+        }
+
+        public Builder setTypes(List<String> types) {
+            this.types = Objects.requireNonNull(types, TYPES.getPreferredName());
+            return this;
+        }
+
+        public Builder setQueryDelay(TimeValue queryDelay) {
+            this.queryDelay = queryDelay;
+            return this;
+        }
+
+        public Builder setFrequency(TimeValue frequency) {
+            this.frequency = frequency;
+            return this;
+        }
+
+        public Builder setQuery(QueryBuilder query) {
+            this.query = Objects.requireNonNull(query, QUERY.getPreferredName());
+            return this;
+        }
+
+        public Builder setAggregations(AggregatorFactories.Builder aggregations) {
+            this.aggregations = aggregations;
+            return this;
+        }
+
+        public Builder setScriptFields(List<SearchSourceBuilder.ScriptField> scriptFields) {
+            List<SearchSourceBuilder.ScriptField> sorted = new ArrayList<>(scriptFields);
+            sorted.sort(Comparator.comparing(SearchSourceBuilder.ScriptField::fieldName));
+            this.scriptFields = sorted;
+            return this;
+        }
+
+        public Builder setScrollSize(int scrollSize) {
+            this.scrollSize = scrollSize;
+            return this;
+        }
+
+        public Builder setChunkingConfig(ChunkingConfig chunkingConfig) {
+            this.chunkingConfig = chunkingConfig;
+            return this;
+        }
+
+        public DatafeedConfig build() {
+            return new DatafeedConfig(id, jobId, queryDelay, frequency, indices, types, query, aggregations, scriptFields, scrollSize,
+                chunkingConfig);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedUpdate.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedUpdate.java
new file mode 100644
index 0000000000000..787bdf06e5ec2
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedUpdate.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.datafeed;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A datafeed update contains partial properties to update a {@link DatafeedConfig}.
+ * The main difference between this class and {@link DatafeedConfig} is that here all
+ * fields are nullable.
+ */
+public class DatafeedUpdate implements ToXContentObject {
+
+    public static final ConstructingObjectParser<Builder, Void> PARSER = new ConstructingObjectParser<>(
+        "datafeed_update", true, a -> new Builder((String)a[0]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), DatafeedConfig.ID);
+
+        PARSER.declareString(Builder::setJobId, Job.ID);
+        PARSER.declareStringArray(Builder::setIndices, DatafeedConfig.INDEXES);
+        PARSER.declareStringArray(Builder::setIndices, DatafeedConfig.INDICES);
+        PARSER.declareStringArray(Builder::setTypes, DatafeedConfig.TYPES);
+        PARSER.declareString((builder, val) -> builder.setQueryDelay(
+            TimeValue.parseTimeValue(val, DatafeedConfig.QUERY_DELAY.getPreferredName())), DatafeedConfig.QUERY_DELAY);
+        PARSER.declareString((builder, val) -> builder.setFrequency(
+            TimeValue.parseTimeValue(val, DatafeedConfig.FREQUENCY.getPreferredName())), DatafeedConfig.FREQUENCY);
+        PARSER.declareObject(Builder::setQuery, (p, c) -> AbstractQueryBuilder.parseInnerQueryBuilder(p), DatafeedConfig.QUERY);
+        PARSER.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p),
+            DatafeedConfig.AGGREGATIONS);
+        PARSER.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p),
+            DatafeedConfig.AGGS);
+        PARSER.declareObject(Builder::setScriptFields, (p, c) -> {
+            List<SearchSourceBuilder.ScriptField> parsedScriptFields = new ArrayList<>();
+            while (p.nextToken() != XContentParser.Token.END_OBJECT) {
+                parsedScriptFields.add(new SearchSourceBuilder.ScriptField(p));
+            }
+            return parsedScriptFields;
+        }, DatafeedConfig.SCRIPT_FIELDS);
+        PARSER.declareInt(Builder::setScrollSize, DatafeedConfig.SCROLL_SIZE);
+        PARSER.declareObject(Builder::setChunkingConfig, ChunkingConfig.PARSER, DatafeedConfig.CHUNKING_CONFIG);
+    }
+
+    private final String id;
+    private final String jobId;
+    private final TimeValue queryDelay;
+    private final TimeValue frequency;
+    private final List<String> indices;
+    private final List<String> types;
+    private final QueryBuilder query;
+    private final AggregatorFactories.Builder aggregations;
+    private final List<SearchSourceBuilder.ScriptField> scriptFields;
+    private final Integer scrollSize;
+    private final ChunkingConfig chunkingConfig;
+
+    private DatafeedUpdate(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List<String> indices, List<String> types,
+                           QueryBuilder query, AggregatorFactories.Builder aggregations, List<SearchSourceBuilder.ScriptField> scriptFields,
+                           Integer scrollSize, ChunkingConfig chunkingConfig) {
+        this.id = id;
+        this.jobId = jobId;
+        this.queryDelay = queryDelay;
+        this.frequency = frequency;
+        this.indices = indices;
+        this.types = types;
+        this.query = query;
+        this.aggregations = aggregations;
+        this.scriptFields = scriptFields;
+        this.scrollSize = scrollSize;
+        this.chunkingConfig = chunkingConfig;
+    }
+
+    /**
+     * Get the id of the datafeed to update
+     */
+    public String getId() {
+        return id;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(DatafeedConfig.ID.getPreferredName(), id);
+        addOptionalField(builder, Job.ID, jobId);
+        if (queryDelay != null) {
+            builder.field(DatafeedConfig.QUERY_DELAY.getPreferredName(), queryDelay.getStringRep());
+        }
+        if (frequency != null) {
+            builder.field(DatafeedConfig.FREQUENCY.getPreferredName(), frequency.getStringRep());
+        }
+        addOptionalField(builder, DatafeedConfig.INDICES, indices);
+        addOptionalField(builder, DatafeedConfig.TYPES, types);
+        addOptionalField(builder, DatafeedConfig.QUERY, query);
+        addOptionalField(builder, DatafeedConfig.AGGREGATIONS, aggregations);
+        if (scriptFields != null) {
+            builder.startObject(DatafeedConfig.SCRIPT_FIELDS.getPreferredName());
+            for (SearchSourceBuilder.ScriptField scriptField : scriptFields) {
+                scriptField.toXContent(builder, params);
+            }
+            builder.endObject();
+        }
+        addOptionalField(builder, DatafeedConfig.SCROLL_SIZE, scrollSize);
+        addOptionalField(builder, DatafeedConfig.CHUNKING_CONFIG, chunkingConfig);
+        builder.endObject();
+        return builder;
+    }
+
+    private void addOptionalField(XContentBuilder builder, ParseField field, Object value) throws IOException {
+        if (value != null) {
+            builder.field(field.getPreferredName(), value);
+        }
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public TimeValue getQueryDelay() {
+        return queryDelay;
+    }
+
+    public TimeValue getFrequency() {
+        return frequency;
+    }
+
+    public List<String> getIndices() {
+        return indices;
+    }
+
+    public List<String> getTypes() {
+        return types;
+    }
+
+    public Integer getScrollSize() {
+        return scrollSize;
+    }
+
+    public QueryBuilder getQuery() {
+        return query;
+    }
+
+    public AggregatorFactories.Builder getAggregations() {
+        return aggregations;
+    }
+
+    public List<SearchSourceBuilder.ScriptField> getScriptFields() {
+        return scriptFields == null ? Collections.emptyList() : scriptFields;
+    }
+
+    public ChunkingConfig getChunkingConfig() {
+        return chunkingConfig;
+    }
+
+    /**
+     * The lists of indices and types are compared for equality but they are not
+     * sorted first so this test could fail simply because the indices and types
+     * lists are in different orders.
+     */
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        DatafeedUpdate that = (DatafeedUpdate) other;
+
+        return Objects.equals(this.id, that.id)
+            && Objects.equals(this.jobId, that.jobId)
+            && Objects.equals(this.frequency, that.frequency)
+            && Objects.equals(this.queryDelay, that.queryDelay)
+            && Objects.equals(this.indices, that.indices)
+            && Objects.equals(this.types, that.types)
+            && Objects.equals(this.query, that.query)
+            && Objects.equals(this.scrollSize, that.scrollSize)
+            && Objects.equals(this.aggregations, that.aggregations)
+            && Objects.equals(this.scriptFields, that.scriptFields)
+            && Objects.equals(this.chunkingConfig, that.chunkingConfig);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, jobId, frequency, queryDelay, indices, types, query, scrollSize, aggregations, scriptFields,
+            chunkingConfig);
+    }
+
+    public static class Builder {
+
+        private String id;
+        private String jobId;
+        private TimeValue queryDelay;
+        private TimeValue frequency;
+        private List<String> indices;
+        private List<String> types;
+        private QueryBuilder query;
+        private AggregatorFactories.Builder aggregations;
+        private List<SearchSourceBuilder.ScriptField> scriptFields;
+        private Integer scrollSize;
+        private ChunkingConfig chunkingConfig;
+
+        public Builder(String id) {
+            this.id = Objects.requireNonNull(id, DatafeedConfig.ID.getPreferredName());
+        }
+
+        public Builder(DatafeedUpdate config) {
+            this.id = config.id;
+            this.jobId = config.jobId;
+            this.queryDelay = config.queryDelay;
+            this.frequency = config.frequency;
+            this.indices = config.indices;
+            this.types = config.types;
+            this.query = config.query;
+            this.aggregations = config.aggregations;
+            this.scriptFields = config.scriptFields;
+            this.scrollSize = config.scrollSize;
+            this.chunkingConfig = config.chunkingConfig;
+        }
+
+        public Builder setJobId(String jobId) {
+            this.jobId = jobId;
+            return this;
+        }
+
+        public Builder setIndices(List<String> indices) {
+            this.indices = indices;
+            return this;
+        }
+
+        public Builder setTypes(List<String> types) {
+            this.types = types;
+            return this;
+        }
+
+        public Builder setQueryDelay(TimeValue queryDelay) {
+            this.queryDelay = queryDelay;
+            return this;
+        }
+
+        public Builder setFrequency(TimeValue frequency) {
+            this.frequency = frequency;
+            return this;
+        }
+
+        public Builder setQuery(QueryBuilder query) {
+            this.query = query;
+            return this;
+        }
+
+        public Builder setAggregations(AggregatorFactories.Builder aggregations) {
+            this.aggregations = aggregations;
+            return this;
+        }
+
+        public Builder setScriptFields(List<SearchSourceBuilder.ScriptField> scriptFields) {
+            List<SearchSourceBuilder.ScriptField> sorted = new ArrayList<>(scriptFields);
+            sorted.sort(Comparator.comparing(SearchSourceBuilder.ScriptField::fieldName));
+            this.scriptFields = sorted;
+            return this;
+        }
+
+        public Builder setScrollSize(int scrollSize) {
+            this.scrollSize = scrollSize;
+            return this;
+        }
+
+        public Builder setChunkingConfig(ChunkingConfig chunkingConfig) {
+            this.chunkingConfig = chunkingConfig;
+            return this;
+        }
+
+        public DatafeedUpdate build() {
+            return new DatafeedUpdate(id, jobId, queryDelay, frequency, indices, types, query, aggregations, scriptFields, scrollSize,
+                chunkingConfig);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfig.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfig.java
new file mode 100644
index 0000000000000..1c106f782086e
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfig.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+
+/**
+ * Analysis configuration options that describe which fields are
+ * analyzed and which functions are used to detect anomalies.
+ * <p>
+ * The configuration can contain multiple detectors, a new anomaly detector will
+ * be created for each detector configuration. The fields
+ * <code>bucketSpan, summaryCountFieldName and categorizationFieldName</code>
+ * apply to all detectors.
+ * <p>
+ * If a value has not been set it will be <code>null</code>
+ * Object wrappers are used around integral types &amp; booleans so they can take
+ * <code>null</code> values.
+ */
+public class AnalysisConfig implements ToXContentObject {
+    /**
+     * Serialisation names
+     */
+    public static final ParseField ANALYSIS_CONFIG = new ParseField("analysis_config");
+    public static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
+    public static final ParseField CATEGORIZATION_FIELD_NAME = new ParseField("categorization_field_name");
+    public static final ParseField CATEGORIZATION_FILTERS = new ParseField("categorization_filters");
+    public static final ParseField CATEGORIZATION_ANALYZER = CategorizationAnalyzerConfig.CATEGORIZATION_ANALYZER;
+    public static final ParseField LATENCY = new ParseField("latency");
+    public static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name");
+    public static final ParseField DETECTORS = new ParseField("detectors");
+    public static final ParseField INFLUENCERS = new ParseField("influencers");
+    public static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets");
+    public static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window");
+    public static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<Builder, Void> PARSER = new ConstructingObjectParser<>(ANALYSIS_CONFIG.getPreferredName(),
+        true, a -> new AnalysisConfig.Builder((List<Detector>) a[0]));
+
+    static {
+        PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(),
+            (p, c) -> (Detector.PARSER).apply(p, c).build(), DETECTORS);
+        PARSER.declareString((builder, val) ->
+            builder.setBucketSpan(TimeValue.parseTimeValue(val, BUCKET_SPAN.getPreferredName())), BUCKET_SPAN);
+        PARSER.declareString(Builder::setCategorizationFieldName, CATEGORIZATION_FIELD_NAME);
+        PARSER.declareStringArray(Builder::setCategorizationFilters, CATEGORIZATION_FILTERS);
+        // This one is nasty - the syntax for analyzers takes either names or objects at many levels, hence it's not
+        // possible to simply declare whether the field is a string or object and a completely custom parser is required
+        PARSER.declareField(Builder::setCategorizationAnalyzerConfig,
+            (p, c) -> CategorizationAnalyzerConfig.buildFromXContentFragment(p),
+            CATEGORIZATION_ANALYZER, ObjectParser.ValueType.OBJECT_OR_STRING);
+        PARSER.declareString((builder, val) ->
+            builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY);
+        PARSER.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME);
+        PARSER.declareStringArray(Builder::setInfluencers, INFLUENCERS);
+        PARSER.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS);
+        PARSER.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW);
+        PARSER.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);
+    }
+
+    /**
+     * These values apply to all detectors
+     */
+    private final TimeValue bucketSpan;
+    private final String categorizationFieldName;
+    private final List<String> categorizationFilters;
+    private final CategorizationAnalyzerConfig categorizationAnalyzerConfig;
+    private final TimeValue latency;
+    private final String summaryCountFieldName;
+    private final List<Detector> detectors;
+    private final List<String> influencers;
+    private final Boolean overlappingBuckets;
+    private final Long resultFinalizationWindow;
+    private final Boolean multivariateByFields;
+
+    private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List<String> categorizationFilters,
+                           CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName,
+                           List<Detector> detectors, List<String> influencers, Boolean overlappingBuckets, Long resultFinalizationWindow,
+                           Boolean multivariateByFields) {
+        this.detectors = Collections.unmodifiableList(detectors);
+        this.bucketSpan = bucketSpan;
+        this.latency = latency;
+        this.categorizationFieldName = categorizationFieldName;
+        this.categorizationAnalyzerConfig = categorizationAnalyzerConfig;
+        this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters);
+        this.summaryCountFieldName = summaryCountFieldName;
+        this.influencers = Collections.unmodifiableList(influencers);
+        this.overlappingBuckets = overlappingBuckets;
+        this.resultFinalizationWindow = resultFinalizationWindow;
+        this.multivariateByFields = multivariateByFields;
+    }
+
+    /**
+     * The analysis bucket span
+     *
+     * @return The bucketspan or <code>null</code> if not set
+     */
+    public TimeValue getBucketSpan() {
+        return bucketSpan;
+    }
+
+    public String getCategorizationFieldName() {
+        return categorizationFieldName;
+    }
+
+    public List<String> getCategorizationFilters() {
+        return categorizationFilters;
+    }
+
+    public CategorizationAnalyzerConfig getCategorizationAnalyzerConfig() {
+        return categorizationAnalyzerConfig;
+    }
+
+    /**
+     * The latency interval during which out-of-order records should be handled.
+     *
+     * @return The latency interval or <code>null</code> if not set
+     */
+    public TimeValue getLatency() {
+        return latency;
+    }
+
+    /**
+     * The name of the field that contains counts for pre-summarised input
+     *
+     * @return The field name or <code>null</code> if not set
+     */
+    public String getSummaryCountFieldName() {
+        return summaryCountFieldName;
+    }
+
+    /**
+     * The list of analysis detectors. In a valid configuration the list should
+     * contain at least 1 {@link Detector}
+     *
+     * @return The Detectors used in this job
+     */
+    public List<Detector> getDetectors() {
+        return detectors;
+    }
+
+    /**
+     * The list of influence field names
+     */
+    public List<String> getInfluencers() {
+        return influencers;
+    }
+
+    public Boolean getOverlappingBuckets() {
+        return overlappingBuckets;
+    }
+
+    public Long getResultFinalizationWindow() {
+        return resultFinalizationWindow;
+    }
+
+    public Boolean getMultivariateByFields() {
+        return multivariateByFields;
+    }
+
+    private static void addIfNotNull(Set<String> fields, String field) {
+        if (field != null) {
+            fields.add(field);
+        }
+    }
+
+    public List<String> fields() {
+        return collectNonNullAndNonEmptyDetectorFields(Detector::getFieldName);
+    }
+
+    private List<String> collectNonNullAndNonEmptyDetectorFields(
+        Function<Detector, String> fieldGetter) {
+        Set<String> fields = new HashSet<>();
+
+        for (Detector d : getDetectors()) {
+            addIfNotNull(fields, fieldGetter.apply(d));
+        }
+
+        // remove empty strings
+        fields.remove("");
+
+        return new ArrayList<>(fields);
+    }
+
+    public List<String> byFields() {
+        return collectNonNullAndNonEmptyDetectorFields(Detector::getByFieldName);
+    }
+
+    public List<String> overFields() {
+        return collectNonNullAndNonEmptyDetectorFields(Detector::getOverFieldName);
+    }
+
+    public List<String> partitionFields() {
+        return collectNonNullAndNonEmptyDetectorFields(Detector::getPartitionFieldName);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (bucketSpan != null) {
+            builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan.getStringRep());
+        }
+        if (categorizationFieldName != null) {
+            builder.field(CATEGORIZATION_FIELD_NAME.getPreferredName(), categorizationFieldName);
+        }
+        if (categorizationFilters != null) {
+            builder.field(CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters);
+        }
+        if (categorizationAnalyzerConfig != null) {
+            // This cannot be builder.field(CATEGORIZATION_ANALYZER.getPreferredName(), categorizationAnalyzerConfig, params);
+            // because that always writes categorizationAnalyzerConfig as an object, and in the case of a global analyzer it
+            // gets written as a single string.
+            categorizationAnalyzerConfig.toXContent(builder, params);
+        }
+        if (latency != null) {
+            builder.field(LATENCY.getPreferredName(), latency.getStringRep());
+        }
+        if (summaryCountFieldName != null) {
+            builder.field(SUMMARY_COUNT_FIELD_NAME.getPreferredName(), summaryCountFieldName);
+        }
+        builder.startArray(DETECTORS.getPreferredName());
+        for (Detector detector : detectors) {
+            detector.toXContent(builder, params);
+        }
+        builder.endArray();
+        builder.field(INFLUENCERS.getPreferredName(), influencers);
+        if (overlappingBuckets != null) {
+            builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets);
+        }
+        if (resultFinalizationWindow != null) {
+            builder.field(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow);
+        }
+        if (multivariateByFields != null) {
+            builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) {
+            return true;
+        }
+
+        if (object == null || getClass() != object.getClass()) {
+            return false;
+        }
+
+        AnalysisConfig that = (AnalysisConfig) object;
+        return Objects.equals(latency, that.latency) &&
+            Objects.equals(bucketSpan, that.bucketSpan) &&
+            Objects.equals(categorizationFieldName, that.categorizationFieldName) &&
+            Objects.equals(categorizationFilters, that.categorizationFilters) &&
+            Objects.equals(categorizationAnalyzerConfig, that.categorizationAnalyzerConfig) &&
+            Objects.equals(summaryCountFieldName, that.summaryCountFieldName) &&
+            Objects.equals(detectors, that.detectors) &&
+            Objects.equals(influencers, that.influencers) &&
+            Objects.equals(overlappingBuckets, that.overlappingBuckets) &&
+            Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) &&
+            Objects.equals(multivariateByFields, that.multivariateByFields);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency,
+            summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow,
+            multivariateByFields);
+    }
+
+    public static class Builder {
+
+        private List<Detector> detectors;
+        private TimeValue bucketSpan;
+        private TimeValue latency;
+        private String categorizationFieldName;
+        private List<String> categorizationFilters;
+        private CategorizationAnalyzerConfig categorizationAnalyzerConfig;
+        private String summaryCountFieldName;
+        private List<String> influencers = new ArrayList<>();
+        private Boolean overlappingBuckets;
+        private Long resultFinalizationWindow;
+        private Boolean multivariateByFields;
+
+        public Builder(List<Detector> detectors) {
+            setDetectors(detectors);
+        }
+
+        public Builder(AnalysisConfig analysisConfig) {
+            this.detectors = new ArrayList<>(analysisConfig.detectors);
+            this.bucketSpan = analysisConfig.bucketSpan;
+            this.latency = analysisConfig.latency;
+            this.categorizationFieldName = analysisConfig.categorizationFieldName;
+            this.categorizationFilters = analysisConfig.categorizationFilters == null ? null
+                : new ArrayList<>(analysisConfig.categorizationFilters);
+            this.categorizationAnalyzerConfig = analysisConfig.categorizationAnalyzerConfig;
+            this.summaryCountFieldName = analysisConfig.summaryCountFieldName;
+            this.influencers = new ArrayList<>(analysisConfig.influencers);
+            this.overlappingBuckets = analysisConfig.overlappingBuckets;
+            this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow;
+            this.multivariateByFields = analysisConfig.multivariateByFields;
+        }
+
+        public void setDetectors(List<Detector> detectors) {
+            Objects.requireNonNull(detectors,  "[" + DETECTORS.getPreferredName() + "] must not be null");
+            // We always assign sequential IDs to the detectors that are correct for this analysis config
+            int detectorIndex = 0;
+            List<Detector> sequentialIndexDetectors = new ArrayList<>(detectors.size());
+            for (Detector origDetector : detectors) {
+                Detector.Builder builder = new Detector.Builder(origDetector);
+                builder.setDetectorIndex(detectorIndex++);
+                sequentialIndexDetectors.add(builder.build());
+            }
+            this.detectors = sequentialIndexDetectors;
+        }
+
+        public void setDetector(int detectorIndex, Detector detector) {
+            detectors.set(detectorIndex, detector);
+        }
+
+        public void setBucketSpan(TimeValue bucketSpan) {
+            this.bucketSpan = bucketSpan;
+        }
+
+        public void setLatency(TimeValue latency) {
+            this.latency = latency;
+        }
+
+        public void setCategorizationFieldName(String categorizationFieldName) {
+            this.categorizationFieldName = categorizationFieldName;
+        }
+
+        public void setCategorizationFilters(List<String> categorizationFilters) {
+            this.categorizationFilters = categorizationFilters;
+        }
+
+        public void setCategorizationAnalyzerConfig(CategorizationAnalyzerConfig categorizationAnalyzerConfig) {
+            this.categorizationAnalyzerConfig = categorizationAnalyzerConfig;
+        }
+
+        public void setSummaryCountFieldName(String summaryCountFieldName) {
+            this.summaryCountFieldName = summaryCountFieldName;
+        }
+
+        public void setInfluencers(List<String> influencers) {
+            this.influencers = Objects.requireNonNull(influencers, INFLUENCERS.getPreferredName());
+        }
+
+        public void setOverlappingBuckets(Boolean overlappingBuckets) {
+            this.overlappingBuckets = overlappingBuckets;
+        }
+
+        public void setResultFinalizationWindow(Long resultFinalizationWindow) {
+            this.resultFinalizationWindow = resultFinalizationWindow;
+        }
+
+        public void setMultivariateByFields(Boolean multivariateByFields) {
+            this.multivariateByFields = multivariateByFields;
+        }
+
+        public AnalysisConfig build() {
+
+            return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig,
+                latency, summaryCountFieldName, detectors, influencers, overlappingBuckets,
+                resultFinalizationWindow, multivariateByFields);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisLimits.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisLimits.java
new file mode 100644
index 0000000000000..f69b9ccbf9ff4
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisLimits.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Analysis limits for autodetect. In particular,
+ * this is a collection of parameters that allow limiting
+ * the resources used by the job.
+ */
+public class AnalysisLimits implements ToXContentObject {
+
+    /**
+     * Serialisation field names
+     */
+    public static final ParseField MODEL_MEMORY_LIMIT = new ParseField("model_memory_limit");
+    public static final ParseField CATEGORIZATION_EXAMPLES_LIMIT = new ParseField("categorization_examples_limit");
+
+    public static final ConstructingObjectParser<AnalysisLimits, Void> PARSER =
+        new ConstructingObjectParser<>("analysis_limits", true, a -> new AnalysisLimits((Long) a[0], (Long) a[1]));
+
+    static {
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return ByteSizeValue.parseBytesSizeValue(p.text(), MODEL_MEMORY_LIMIT.getPreferredName()).getMb();
+            } else if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+                return p.longValue();
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, MODEL_MEMORY_LIMIT, ObjectParser.ValueType.VALUE);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), CATEGORIZATION_EXAMPLES_LIMIT);
+    }
+
+    /**
+     * The model memory limit in MiBs.
+     * It is initialised to <code>null</code>, which implies that the server-side default will be used.
+     */
+    private final Long modelMemoryLimit;
+
+    /**
+     * It is initialised to <code>null</code>.
+     * A value of <code>null</code> will result in the server-side default being used.
+     */
+    private final Long categorizationExamplesLimit;
+
+    public AnalysisLimits(Long categorizationExamplesLimit) {
+        this(null, categorizationExamplesLimit);
+    }
+
+    public AnalysisLimits(Long modelMemoryLimit, Long categorizationExamplesLimit) {
+        this.modelMemoryLimit = modelMemoryLimit;
+        this.categorizationExamplesLimit = categorizationExamplesLimit;
+    }
+
+    /**
+     * Maximum size of the model in MB before the anomaly detector
+     * will drop new samples to prevent the model using any more
+     * memory.
+     *
+     * @return The set memory limit or <code>null</code> if not set
+     */
+    @Nullable
+    public Long getModelMemoryLimit() {
+        return modelMemoryLimit;
+    }
+
+    /**
+     * Gets the limit to the number of examples that are stored per category
+     *
+     * @return the limit or <code>null</code> if not set
+     */
+    @Nullable
+    public Long getCategorizationExamplesLimit() {
+        return categorizationExamplesLimit;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (modelMemoryLimit != null) {
+            builder.field(MODEL_MEMORY_LIMIT.getPreferredName(), modelMemoryLimit + "mb");
+        }
+        if (categorizationExamplesLimit != null) {
+            builder.field(CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName(), categorizationExamplesLimit);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * Overridden equality test
+     */
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other instanceof AnalysisLimits == false) {
+            return false;
+        }
+
+        AnalysisLimits that = (AnalysisLimits) other;
+        return Objects.equals(this.modelMemoryLimit, that.modelMemoryLimit) &&
+                Objects.equals(this.categorizationExamplesLimit, that.categorizationExamplesLimit);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(modelMemoryLimit, categorizationExamplesLimit);
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/CategorizationAnalyzerConfig.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/CategorizationAnalyzerConfig.java
new file mode 100644
index 0000000000000..dc7f047b80404
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/CategorizationAnalyzerConfig.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.action.admin.indices.RestAnalyzeAction;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Configuration for the categorization analyzer.
+ *
+ * The syntax is a subset of what can be supplied to the {@linkplain RestAnalyzeAction <code>_analyze</code> endpoint}.
+ * To summarize, the first option is to specify the name of an out-of-the-box analyzer:
+ * <code>
+ *     "categorization_analyzer" : "standard"
+ * </code>
+ *
+ * The second option is to specify a custom analyzer by combining the <code>char_filters</code>, <code>tokenizer</code>
+ * and <code>token_filters</code> fields.  In turn, each of these can be specified as the name of an out-of-the-box
+ * one or as an object defining a custom one.  For example:
+ * <code>
+ *     "char_filters" : [
+ *         "html_strip",
+ *         { "type" : "pattern_replace", "pattern": "SQL: .*" }
+ *     ],
+ *     "tokenizer" : "thai",
+ *     "token_filters" : [
+ *         "lowercase",
+ *         { "type" : "pattern_replace", "pattern": "^[0-9].*" }
+ *     ]
+ * </code>
+ */
+public class CategorizationAnalyzerConfig implements ToXContentFragment {
+
+    public static final ParseField CATEGORIZATION_ANALYZER = new ParseField("categorization_analyzer");
+    private static final ParseField TOKENIZER = RestAnalyzeAction.Fields.TOKENIZER;
+    private static final ParseField TOKEN_FILTERS = RestAnalyzeAction.Fields.TOKEN_FILTERS;
+    private static final ParseField CHAR_FILTERS = RestAnalyzeAction.Fields.CHAR_FILTERS;
+
+    /**
+     * This method is only used in the unit tests - in production code this config is always parsed as a fragment.
+     */
+    static CategorizationAnalyzerConfig buildFromXContentObject(XContentParser parser) throws IOException {
+
+        if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
+            throw new IllegalArgumentException("Expected start object but got [" + parser.currentToken() + "]");
+        }
+        if (parser.nextToken() != XContentParser.Token.FIELD_NAME) {
+            throw new IllegalArgumentException("Expected field name but got [" + parser.currentToken() + "]");
+        }
+        parser.nextToken();
+        CategorizationAnalyzerConfig categorizationAnalyzerConfig = buildFromXContentFragment(parser);
+        parser.nextToken();
+        return categorizationAnalyzerConfig;
+    }
+
+    /**
+     * Parse a <code>categorization_analyzer</code> configuration.  A custom parser is needed due to the
+     * complexity of the format, with many elements able to be specified as either the name of a built-in
+     * element or an object containing a custom definition.
+     */
+    static CategorizationAnalyzerConfig buildFromXContentFragment(XContentParser parser) throws IOException {
+
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder();
+
+        XContentParser.Token token = parser.currentToken();
+        if (token == XContentParser.Token.VALUE_STRING) {
+            builder.setAnalyzer(parser.text());
+        } else if (token != XContentParser.Token.START_OBJECT) {
+            throw new IllegalArgumentException("[" + CATEGORIZATION_ANALYZER + "] should be analyzer's name or settings [" + token + "]");
+        } else {
+            String currentFieldName = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (CHAR_FILTERS.match(currentFieldName, parser.getDeprecationHandler())
+                        && token == XContentParser.Token.START_ARRAY) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token == XContentParser.Token.VALUE_STRING) {
+                            builder.addCharFilter(parser.text());
+                        } else if (token == XContentParser.Token.START_OBJECT) {
+                            builder.addCharFilter(parser.map());
+                        } else {
+                            throw new IllegalArgumentException("[" + currentFieldName + "] in [" + CATEGORIZATION_ANALYZER +
+                                    "] array element should contain char_filter's name or settings [" + token + "]");
+                        }
+                    }
+                } else if (TOKENIZER.match(currentFieldName, parser.getDeprecationHandler())) {
+                    if (token == XContentParser.Token.VALUE_STRING) {
+                        builder.setTokenizer(parser.text());
+                    } else if (token == XContentParser.Token.START_OBJECT) {
+                        builder.setTokenizer(parser.map());
+                    } else {
+                        throw new IllegalArgumentException("[" + currentFieldName + "] in [" + CATEGORIZATION_ANALYZER +
+                                "] should be tokenizer's name or settings [" + token + "]");
+                    }
+                } else if (TOKEN_FILTERS.match(currentFieldName, parser.getDeprecationHandler())
+                        && token == XContentParser.Token.START_ARRAY) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token == XContentParser.Token.VALUE_STRING) {
+                            builder.addTokenFilter(parser.text());
+                        } else if (token == XContentParser.Token.START_OBJECT) {
+                            builder.addTokenFilter(parser.map());
+                        } else {
+                            throw new IllegalArgumentException("[" + currentFieldName + "] in [" + CATEGORIZATION_ANALYZER +
+                                    "] array element should contain token_filter's name or settings [" + token + "]");
+                        }
+                    }
+                }
+            }
+        }
+
+        return builder.build();
+    }
+
+    /**
+     * Simple store of either a name of a built-in analyzer element or a custom definition.
+     */
+    public static final class NameOrDefinition implements ToXContentFragment {
+
+        // Exactly one of these two members is not null
+        public final String name;
+        public final Settings definition;
+
+        NameOrDefinition(String name) {
+            this.name = Objects.requireNonNull(name);
+            this.definition = null;
+        }
+
+        NameOrDefinition(ParseField field, Map<String, Object> definition) {
+            this.name = null;
+            Objects.requireNonNull(definition);
+            try {
+                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+                builder.map(definition);
+                this.definition = Settings.builder().loadFromSource(Strings.toString(builder), builder.contentType()).build();
+            } catch (IOException e) {
+                throw new IllegalArgumentException("Failed to parse [" + definition + "] in [" + field.getPreferredName() + "]", e);
+            }
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            if (definition == null) {
+                builder.value(name);
+            } else {
+                builder.startObject();
+                definition.toXContent(builder, params);
+                builder.endObject();
+            }
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            NameOrDefinition that = (NameOrDefinition) o;
+            return Objects.equals(name, that.name) &&
+                    Objects.equals(definition, that.definition);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(name, definition);
+        }
+
+        @Override
+        public String toString() {
+            if (definition == null) {
+                return name;
+            } else {
+                return definition.toDelimitedString(';');
+            }
+        }
+    }
+
+    private final String analyzer;
+    private final List<NameOrDefinition> charFilters;
+    private final NameOrDefinition tokenizer;
+    private final List<NameOrDefinition> tokenFilters;
+
+    private CategorizationAnalyzerConfig(String analyzer, List<NameOrDefinition> charFilters, NameOrDefinition tokenizer,
+                                         List<NameOrDefinition> tokenFilters) {
+        this.analyzer = analyzer;
+        this.charFilters = Collections.unmodifiableList(charFilters);
+        this.tokenizer = tokenizer;
+        this.tokenFilters = Collections.unmodifiableList(tokenFilters);
+    }
+
+    public String getAnalyzer() {
+        return analyzer;
+    }
+
+    public List<NameOrDefinition> getCharFilters() {
+        return charFilters;
+    }
+
+    public NameOrDefinition getTokenizer() {
+        return tokenizer;
+    }
+
+    public List<NameOrDefinition> getTokenFilters() {
+        return tokenFilters;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (analyzer != null) {
+            builder.field(CATEGORIZATION_ANALYZER.getPreferredName(), analyzer);
+        } else {
+            builder.startObject(CATEGORIZATION_ANALYZER.getPreferredName());
+            if (charFilters.isEmpty() == false) {
+                builder.startArray(CHAR_FILTERS.getPreferredName());
+                for (NameOrDefinition charFilter : charFilters) {
+                    charFilter.toXContent(builder, params);
+                }
+                builder.endArray();
+            }
+            if (tokenizer != null) {
+                builder.field(TOKENIZER.getPreferredName(), tokenizer);
+            }
+            if (tokenFilters.isEmpty() == false) {
+                builder.startArray(TOKEN_FILTERS.getPreferredName());
+                for (NameOrDefinition tokenFilter : tokenFilters) {
+                    tokenFilter.toXContent(builder, params);
+                }
+                builder.endArray();
+            }
+            builder.endObject();
+        }
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        CategorizationAnalyzerConfig that = (CategorizationAnalyzerConfig) o;
+        return Objects.equals(analyzer, that.analyzer) &&
+                Objects.equals(charFilters, that.charFilters) &&
+                Objects.equals(tokenizer, that.tokenizer) &&
+                Objects.equals(tokenFilters, that.tokenFilters);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(analyzer, charFilters, tokenizer, tokenFilters);
+    }
+
+    public static class Builder {
+
+        private String analyzer;
+        private List<NameOrDefinition> charFilters = new ArrayList<>();
+        private NameOrDefinition tokenizer;
+        private List<NameOrDefinition> tokenFilters = new ArrayList<>();
+
+        public Builder() {
+        }
+
+        public Builder(CategorizationAnalyzerConfig categorizationAnalyzerConfig) {
+            this.analyzer = categorizationAnalyzerConfig.analyzer;
+            this.charFilters = new ArrayList<>(categorizationAnalyzerConfig.charFilters);
+            this.tokenizer = categorizationAnalyzerConfig.tokenizer;
+            this.tokenFilters = new ArrayList<>(categorizationAnalyzerConfig.tokenFilters);
+        }
+
+        public Builder setAnalyzer(String analyzer) {
+            this.analyzer = analyzer;
+            return this;
+        }
+
+        public Builder addCharFilter(String charFilter) {
+            this.charFilters.add(new NameOrDefinition(charFilter));
+            return this;
+        }
+
+        public Builder addCharFilter(Map<String, Object> charFilter) {
+            this.charFilters.add(new NameOrDefinition(CHAR_FILTERS, charFilter));
+            return this;
+        }
+
+        public Builder setTokenizer(String tokenizer) {
+            this.tokenizer = new NameOrDefinition(tokenizer);
+            return this;
+        }
+
+        public Builder setTokenizer(Map<String, Object> tokenizer) {
+            this.tokenizer = new NameOrDefinition(TOKENIZER, tokenizer);
+            return this;
+        }
+
+        public Builder addTokenFilter(String tokenFilter) {
+            this.tokenFilters.add(new NameOrDefinition(tokenFilter));
+            return this;
+        }
+
+        public Builder addTokenFilter(Map<String, Object> tokenFilter) {
+            this.tokenFilters.add(new NameOrDefinition(TOKEN_FILTERS, tokenFilter));
+            return this;
+        }
+
+        /**
+         * Create a config
+         */
+        public CategorizationAnalyzerConfig build() {
+            return new CategorizationAnalyzerConfig(analyzer, charFilters, tokenizer, tokenFilters);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DataDescription.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DataDescription.java
new file mode 100644
index 0000000000000..f469512f64973
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DataDescription.java
@@ -0,0 +1,282 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * Describes the format of the data used in the job and how it should
+ * be interpreted by the ML job.
+ * <p>
+ * {@link #getTimeField()} is the name of the field containing the timestamp and
+ * {@link #getTimeFormat()} is the format code for the date string in as described by
+ * {@link java.time.format.DateTimeFormatter}.
+ */
+public class DataDescription implements ToXContentObject {
+    /**
+     * Enum of the acceptable data formats.
+     */
+    public enum DataFormat {
+        XCONTENT,
+
+        /**
+         * This is deprecated
+         */
+        DELIMITED;
+
+        /**
+         * Case-insensitive from string method.
+         * Works with either XCONTENT, XContent, etc.
+         *
+         * @param value String representation
+         * @return The data format
+         */
+        public static DataFormat forString(String value) {
+            return DataFormat.valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    private static final ParseField DATA_DESCRIPTION_FIELD = new ParseField("data_description");
+    private static final ParseField FORMAT_FIELD = new ParseField("format");
+    private static final ParseField TIME_FIELD_NAME_FIELD = new ParseField("time_field");
+    private static final ParseField TIME_FORMAT_FIELD = new ParseField("time_format");
+    private static final ParseField FIELD_DELIMITER_FIELD = new ParseField("field_delimiter");
+    private static final ParseField QUOTE_CHARACTER_FIELD = new ParseField("quote_character");
+
+    /**
+     * Special time format string for epoch times (seconds)
+     */
+    public static final String EPOCH = "epoch";
+
+    /**
+     * Special time format string for epoch times (milli-seconds)
+     */
+    public static final String EPOCH_MS = "epoch_ms";
+
+    /**
+     * By default autodetect expects the timestamp in a field with this name
+     */
+    public static final String DEFAULT_TIME_FIELD = "time";
+
+    /**
+     * The default field delimiter expected by the native autodetect
+     * program.
+     */
+    public static final char DEFAULT_DELIMITER = '\t';
+
+    /**
+     * The default quote character used to escape text in
+     * delimited data formats
+     */
+    public static final char DEFAULT_QUOTE_CHAR = '"';
+
+    private final DataFormat dataFormat;
+    private final String timeFieldName;
+    private final String timeFormat;
+    private final Character fieldDelimiter;
+    private final Character quoteCharacter;
+
+    public static final ObjectParser<Builder, Void> PARSER =
+        new ObjectParser<>(DATA_DESCRIPTION_FIELD.getPreferredName(), true, Builder::new);
+
+    static {
+        PARSER.declareString(Builder::setFormat, FORMAT_FIELD);
+        PARSER.declareString(Builder::setTimeField, TIME_FIELD_NAME_FIELD);
+        PARSER.declareString(Builder::setTimeFormat, TIME_FORMAT_FIELD);
+        PARSER.declareField(Builder::setFieldDelimiter, DataDescription::extractChar, FIELD_DELIMITER_FIELD, ValueType.STRING);
+        PARSER.declareField(Builder::setQuoteCharacter, DataDescription::extractChar, QUOTE_CHARACTER_FIELD, ValueType.STRING);
+    }
+
+    public DataDescription(DataFormat dataFormat, String timeFieldName, String timeFormat, Character fieldDelimiter,
+                           Character quoteCharacter) {
+        this.dataFormat = dataFormat;
+        this.timeFieldName = timeFieldName;
+        this.timeFormat = timeFormat;
+        this.fieldDelimiter = fieldDelimiter;
+        this.quoteCharacter = quoteCharacter;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (dataFormat != DataFormat.XCONTENT) {
+            builder.field(FORMAT_FIELD.getPreferredName(), dataFormat);
+        }
+        builder.field(TIME_FIELD_NAME_FIELD.getPreferredName(), timeFieldName);
+        builder.field(TIME_FORMAT_FIELD.getPreferredName(), timeFormat);
+        if (fieldDelimiter != null) {
+            builder.field(FIELD_DELIMITER_FIELD.getPreferredName(), String.valueOf(fieldDelimiter));
+        }
+        if (quoteCharacter != null) {
+            builder.field(QUOTE_CHARACTER_FIELD.getPreferredName(), String.valueOf(quoteCharacter));
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * The format of the data to be processed.
+     * Defaults to {@link DataDescription.DataFormat#XCONTENT}
+     *
+     * @return The data format
+     */
+    public DataFormat getFormat() {
+        return dataFormat;
+    }
+
+    /**
+     * The name of the field containing the timestamp
+     *
+     * @return A String if set or <code>null</code>
+     */
+    public String getTimeField() {
+        return timeFieldName;
+    }
+
+    /**
+     * Either {@value #EPOCH}, {@value #EPOCH_MS} or a SimpleDateTime format string.
+     * If not set (is <code>null</code> or an empty string) or set to
+     * {@value #EPOCH_MS} (the default) then the date is assumed to be in
+     * milliseconds from the epoch.
+     *
+     * @return A String if set or <code>null</code>
+     */
+    public String getTimeFormat() {
+        return timeFormat;
+    }
+
+    /**
+     * If the data is in a delimited format with a header e.g. csv or tsv
+     * this is the delimiter character used. This is only applicable if
+     * {@linkplain #getFormat()} is {@link DataDescription.DataFormat#DELIMITED}.
+     * The default value for delimited format is {@value #DEFAULT_DELIMITER}.
+     *
+     * @return A char
+     */
+    public Character getFieldDelimiter() {
+        return fieldDelimiter;
+    }
+
+    /**
+     * The quote character used in delimited formats.
+     * The default value for delimited format is {@value #DEFAULT_QUOTE_CHAR}.
+     *
+     * @return The delimited format quote character
+     */
+    public Character getQuoteCharacter() {
+        return quoteCharacter;
+    }
+
+    private static Character extractChar(XContentParser parser) throws IOException {
+        if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
+            String charStr = parser.text();
+            if (charStr.length() != 1) {
+                throw new IllegalArgumentException("String must be a single character, found [" + charStr + "]");
+            }
+            return charStr.charAt(0);
+        }
+        throw new IllegalArgumentException("Unsupported token [" + parser.currentToken() + "]");
+    }
+
+    /**
+     * Overridden equality test
+     */
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other instanceof DataDescription == false) {
+            return false;
+        }
+
+        DataDescription that = (DataDescription) other;
+
+        return this.dataFormat == that.dataFormat &&
+                Objects.equals(this.quoteCharacter, that.quoteCharacter) &&
+                Objects.equals(this.timeFieldName, that.timeFieldName) &&
+                Objects.equals(this.timeFormat, that.timeFormat) &&
+                Objects.equals(this.fieldDelimiter, that.fieldDelimiter);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(dataFormat, quoteCharacter, timeFieldName, timeFormat, fieldDelimiter);
+    }
+
+    public static class Builder {
+
+        private DataFormat dataFormat = DataFormat.XCONTENT;
+        private String timeFieldName = DEFAULT_TIME_FIELD;
+        private String timeFormat = EPOCH_MS;
+        private Character fieldDelimiter;
+        private Character quoteCharacter;
+
+        public void setFormat(DataFormat format) {
+            dataFormat = Objects.requireNonNull(format);
+        }
+
+        private void setFormat(String format) {
+            setFormat(DataFormat.forString(format));
+        }
+
+        public void setTimeField(String fieldName) {
+            timeFieldName = Objects.requireNonNull(fieldName);
+        }
+
+        public void setTimeFormat(String format) {
+            timeFormat = Objects.requireNonNull(format);
+        }
+
+        public void setFieldDelimiter(Character delimiter) {
+            fieldDelimiter = delimiter;
+        }
+
+        public void setQuoteCharacter(Character value) {
+            quoteCharacter = value;
+        }
+
+        public DataDescription build() {
+            if (dataFormat == DataFormat.DELIMITED) {
+                if (fieldDelimiter == null) {
+                    fieldDelimiter = DEFAULT_DELIMITER;
+                }
+                if (quoteCharacter == null) {
+                    quoteCharacter = DEFAULT_QUOTE_CHAR;
+                }
+            }
+            return new DataDescription(dataFormat, timeFieldName, timeFormat, fieldDelimiter, quoteCharacter);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DefaultDetectorDescription.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DefaultDetectorDescription.java
new file mode 100644
index 0000000000000..081e685fc741b
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DefaultDetectorDescription.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.Strings;
+
+public final class DefaultDetectorDescription {
+    private static final String BY_TOKEN = " by ";
+    private static final String OVER_TOKEN = " over ";
+
+    private static final String USE_NULL_OPTION = " usenull=";
+    private static final String PARTITION_FIELD_OPTION = " partitionfield=";
+    private static final String EXCLUDE_FREQUENT_OPTION = " excludefrequent=";
+
+    private DefaultDetectorDescription() {
+    }
+
+    /**
+     * Returns the default description for the given {@code detector}
+     *
+     * @param detector the {@code Detector} for which a default description is requested
+     * @return the default description
+     */
+    public static String of(Detector detector) {
+        StringBuilder sb = new StringBuilder();
+        appendOn(detector, sb);
+        return sb.toString();
+    }
+
+    /**
+     * Appends to the given {@code StringBuilder} the default description
+     * for the given {@code detector}
+     *
+     * @param detector the {@code Detector} for which a default description is requested
+     * @param sb       the {@code StringBuilder} to append to
+     */
+    public static void appendOn(Detector detector, StringBuilder sb) {
+        if (isNotNullOrEmpty(detector.getFunction().getFullName())) {
+            sb.append(detector.getFunction());
+            if (isNotNullOrEmpty(detector.getFieldName())) {
+                sb.append('(').append(quoteField(detector.getFieldName()))
+                .append(')');
+            }
+        } else if (isNotNullOrEmpty(detector.getFieldName())) {
+            sb.append(quoteField(detector.getFieldName()));
+        }
+
+        if (isNotNullOrEmpty(detector.getByFieldName())) {
+            sb.append(BY_TOKEN).append(quoteField(detector.getByFieldName()));
+        }
+
+        if (isNotNullOrEmpty(detector.getOverFieldName())) {
+            sb.append(OVER_TOKEN).append(quoteField(detector.getOverFieldName()));
+        }
+
+        if (detector.isUseNull()) {
+            sb.append(USE_NULL_OPTION).append(detector.isUseNull());
+        }
+
+        if (isNotNullOrEmpty(detector.getPartitionFieldName())) {
+            sb.append(PARTITION_FIELD_OPTION).append(quoteField(detector.getPartitionFieldName()));
+        }
+
+        if (detector.getExcludeFrequent() != null) {
+            sb.append(EXCLUDE_FREQUENT_OPTION).append(detector.getExcludeFrequent());
+        }
+    }
+
+    private static String quoteField(String field) {
+        if (field.matches("\\w*")) {
+            return field;
+        } else {
+            return "\"" + field.replace("\\", "\\\\").replace("\"", "\\\"") + "\"";
+        }
+    }
+
+    private static boolean isNotNullOrEmpty(String arg) {
+        return !Strings.isNullOrEmpty(arg);
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRule.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRule.java
new file mode 100644
index 0000000000000..9a73afe885b1c
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRule.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Objects;
+
+public class DetectionRule implements ToXContentObject {
+
+    public static final ParseField DETECTION_RULE_FIELD = new ParseField("detection_rule");
+    public static final ParseField ACTIONS_FIELD = new ParseField("actions");
+    public static final ParseField SCOPE_FIELD = new ParseField("scope");
+    public static final ParseField CONDITIONS_FIELD = new ParseField("conditions");
+
+    public static final ObjectParser<Builder, Void> PARSER =
+        new ObjectParser<>(DETECTION_RULE_FIELD.getPreferredName(), true, Builder::new);;
+
+    static {
+        PARSER.declareStringArray(Builder::setActions, ACTIONS_FIELD);
+        PARSER.declareObject(Builder::setScope, RuleScope.parser(), SCOPE_FIELD);
+        PARSER.declareObjectArray(Builder::setConditions, RuleCondition.PARSER, CONDITIONS_FIELD);
+    }
+
+    private final EnumSet<RuleAction> actions;
+    private final RuleScope scope;
+    private final List<RuleCondition> conditions;
+
+    private DetectionRule(EnumSet<RuleAction> actions, RuleScope scope, List<RuleCondition> conditions) {
+        this.actions = Objects.requireNonNull(actions);
+        this.scope = Objects.requireNonNull(scope);
+        this.conditions = Collections.unmodifiableList(conditions);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ACTIONS_FIELD.getPreferredName(), actions);
+        if (scope.isEmpty() == false) {
+            builder.field(SCOPE_FIELD.getPreferredName(), scope);
+        }
+        if (conditions.isEmpty() == false) {
+            builder.field(CONDITIONS_FIELD.getPreferredName(), conditions);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public EnumSet<RuleAction> getActions() {
+        return actions;
+    }
+
+    public RuleScope getScope() {
+        return scope;
+    }
+
+    public List<RuleCondition> getConditions() {
+        return conditions;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj instanceof DetectionRule == false) {
+            return false;
+        }
+
+        DetectionRule other = (DetectionRule) obj;
+        return Objects.equals(actions, other.actions)
+                && Objects.equals(scope, other.scope)
+                && Objects.equals(conditions, other.conditions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(actions, scope, conditions);
+    }
+
+    public static class Builder {
+        private EnumSet<RuleAction> actions = EnumSet.of(RuleAction.SKIP_RESULT);
+        private RuleScope scope = new RuleScope();
+        private List<RuleCondition> conditions = Collections.emptyList();
+
+        public Builder(RuleScope.Builder scope) {
+            this.scope = scope.build();
+        }
+
+        public Builder(List<RuleCondition> conditions) {
+            this.conditions = Objects.requireNonNull(conditions);
+        }
+
+        Builder() {
+        }
+
+        public Builder setActions(List<String> actions) {
+            this.actions.clear();
+            actions.stream().map(RuleAction::fromString).forEach(this.actions::add);
+            return this;
+        }
+
+        public Builder setActions(EnumSet<RuleAction> actions) {
+            this.actions = Objects.requireNonNull(actions, ACTIONS_FIELD.getPreferredName());
+            return this;
+        }
+
+        public Builder setActions(RuleAction... actions) {
+            this.actions.clear();
+            Arrays.stream(actions).forEach(this.actions::add);
+            return this;
+        }
+
+        public Builder setScope(RuleScope scope) {
+            this.scope = Objects.requireNonNull(scope);
+            return this;
+        }
+
+        public Builder setConditions(List<RuleCondition> conditions) {
+            this.conditions = Objects.requireNonNull(conditions);
+            return this;
+        }
+
+        public DetectionRule build() {
+            return new DetectionRule(actions, scope, conditions);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Detector.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Detector.java
new file mode 100644
index 0000000000000..3274b03877f14
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Detector.java
@@ -0,0 +1,362 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * Defines the fields and functions used in the analysis. A combination of <code>field_name</code>,
+ * <code>by_field_name</code> and <code>over_field_name</code> can be used depending on the specific
+ * function chosen. For more information see
+ * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-job-resource.html#ml-detectorconfig">configuring
+ * detectors</a> and <a href="https://www.elastic.co/guide/en/elastic-stack-overview/current/ml-functions.html">detector functions</a>.
+ */
+public class Detector implements ToXContentObject {
+
+    public enum ExcludeFrequent {
+        ALL,
+        NONE,
+        BY,
+        OVER;
+
+        /**
+         * Case-insensitive from string method.
+         * Works with either ALL, All, etc.
+         *
+         * @param value String representation
+         * @return The data format
+         */
+        public static ExcludeFrequent forString(String value) {
+            return valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    public static final ParseField DETECTOR_DESCRIPTION_FIELD = new ParseField("detector_description");
+    public static final ParseField FUNCTION_FIELD = new ParseField("function");
+    public static final ParseField FIELD_NAME_FIELD = new ParseField("field_name");
+    public static final ParseField BY_FIELD_NAME_FIELD = new ParseField("by_field_name");
+    public static final ParseField OVER_FIELD_NAME_FIELD = new ParseField("over_field_name");
+    public static final ParseField PARTITION_FIELD_NAME_FIELD = new ParseField("partition_field_name");
+    public static final ParseField USE_NULL_FIELD = new ParseField("use_null");
+    public static final ParseField EXCLUDE_FREQUENT_FIELD = new ParseField("exclude_frequent");
+    public static final ParseField CUSTOM_RULES_FIELD = new ParseField("custom_rules");
+    public static final ParseField DETECTOR_INDEX = new ParseField("detector_index");
+
+    public static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>("detector", true, Builder::new);
+
+    static {
+        PARSER.declareString(Builder::setDetectorDescription, DETECTOR_DESCRIPTION_FIELD);
+        PARSER.declareString(Builder::setFunction, FUNCTION_FIELD);
+        PARSER.declareString(Builder::setFieldName, FIELD_NAME_FIELD);
+        PARSER.declareString(Builder::setByFieldName, BY_FIELD_NAME_FIELD);
+        PARSER.declareString(Builder::setOverFieldName, OVER_FIELD_NAME_FIELD);
+        PARSER.declareString(Builder::setPartitionFieldName, PARTITION_FIELD_NAME_FIELD);
+        PARSER.declareBoolean(Builder::setUseNull, USE_NULL_FIELD);
+        PARSER.declareField(Builder::setExcludeFrequent, p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return ExcludeFrequent.forString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, EXCLUDE_FREQUENT_FIELD, ObjectParser.ValueType.STRING);
+        PARSER.declareObjectArray(Builder::setRules, (p, c) -> DetectionRule.PARSER.apply(p, c).build(), CUSTOM_RULES_FIELD);
+        PARSER.declareInt(Builder::setDetectorIndex, DETECTOR_INDEX);
+    }
+
+    private final String detectorDescription;
+    private final DetectorFunction function;
+    private final String fieldName;
+    private final String byFieldName;
+    private final String overFieldName;
+    private final String partitionFieldName;
+    private final boolean useNull;
+    private final ExcludeFrequent excludeFrequent;
+    private final List<DetectionRule> rules;
+    private final int detectorIndex;
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(DETECTOR_DESCRIPTION_FIELD.getPreferredName(), detectorDescription);
+        builder.field(FUNCTION_FIELD.getPreferredName(), function);
+        if (fieldName != null) {
+            builder.field(FIELD_NAME_FIELD.getPreferredName(), fieldName);
+        }
+        if (byFieldName != null) {
+            builder.field(BY_FIELD_NAME_FIELD.getPreferredName(), byFieldName);
+        }
+        if (overFieldName != null) {
+            builder.field(OVER_FIELD_NAME_FIELD.getPreferredName(), overFieldName);
+        }
+        if (partitionFieldName != null) {
+            builder.field(PARTITION_FIELD_NAME_FIELD.getPreferredName(), partitionFieldName);
+        }
+        if (useNull) {
+            builder.field(USE_NULL_FIELD.getPreferredName(), useNull);
+        }
+        if (excludeFrequent != null) {
+            builder.field(EXCLUDE_FREQUENT_FIELD.getPreferredName(), excludeFrequent);
+        }
+        if (rules.isEmpty() == false) {
+            builder.field(CUSTOM_RULES_FIELD.getPreferredName(), rules);
+        }
+        // negative means unknown
+        if (detectorIndex >= 0) {
+            builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    private Detector(String detectorDescription, DetectorFunction function, String fieldName, String byFieldName, String overFieldName,
+                     String partitionFieldName, boolean useNull, ExcludeFrequent excludeFrequent, List<DetectionRule> rules,
+                     int detectorIndex) {
+        this.function = function;
+        this.fieldName = fieldName;
+        this.byFieldName = byFieldName;
+        this.overFieldName = overFieldName;
+        this.partitionFieldName = partitionFieldName;
+        this.useNull = useNull;
+        this.excludeFrequent = excludeFrequent;
+        this.rules = Collections.unmodifiableList(rules);
+        this.detectorDescription = detectorDescription != null ? detectorDescription : DefaultDetectorDescription.of(this);
+        this.detectorIndex = detectorIndex;
+    }
+
+    public String getDetectorDescription() {
+        return detectorDescription;
+    }
+
+    /**
+     * The analysis function used e.g. count, rare, min etc.
+     *
+     * @return The function or <code>null</code> if not set
+     */
+    public DetectorFunction getFunction() {
+        return function;
+    }
+
+    /**
+     * The Analysis field
+     *
+     * @return The field to analyse
+     */
+    public String getFieldName() {
+        return fieldName;
+    }
+
+    /**
+     * The 'by' field or <code>null</code> if not set.
+     *
+     * @return The 'by' field
+     */
+    public String getByFieldName() {
+        return byFieldName;
+    }
+
+    /**
+     * The 'over' field or <code>null</code> if not set.
+     *
+     * @return The 'over' field
+     */
+    public String getOverFieldName() {
+        return overFieldName;
+    }
+
+    /**
+     * Segments the analysis along another field to have completely
+     * independent baselines for each instance of partitionfield
+     *
+     * @return The Partition Field
+     */
+    public String getPartitionFieldName() {
+        return partitionFieldName;
+    }
+
+    /**
+     * Where there isn't a value for the 'by' or 'over' field should a new
+     * series be used as the 'null' series.
+     *
+     * @return true if the 'null' series should be created
+     */
+    public boolean isUseNull() {
+        return useNull;
+    }
+
+    /**
+     * Excludes frequently-occuring metrics from the analysis;
+     * can apply to 'by' field, 'over' field, or both
+     *
+     * @return the value that the user set
+     */
+    public ExcludeFrequent getExcludeFrequent() {
+        return excludeFrequent;
+    }
+
+    public List<DetectionRule> getRules() {
+        return rules;
+    }
+
+    /**
+     * @return the detector index or a negative number if unknown
+     */
+    public int getDetectorIndex() {
+        return detectorIndex;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other instanceof Detector == false) {
+            return false;
+        }
+
+        Detector that = (Detector) other;
+
+        return Objects.equals(this.detectorDescription, that.detectorDescription) &&
+                Objects.equals(this.function, that.function) &&
+                Objects.equals(this.fieldName, that.fieldName) &&
+                Objects.equals(this.byFieldName, that.byFieldName) &&
+                Objects.equals(this.overFieldName, that.overFieldName) &&
+                Objects.equals(this.partitionFieldName, that.partitionFieldName) &&
+                Objects.equals(this.useNull, that.useNull) &&
+                Objects.equals(this.excludeFrequent, that.excludeFrequent) &&
+                Objects.equals(this.rules, that.rules) &&
+                this.detectorIndex == that.detectorIndex;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName, useNull,
+                excludeFrequent, rules, detectorIndex);
+    }
+
+    public static class Builder {
+
+        private String detectorDescription;
+        private DetectorFunction function;
+        private String fieldName;
+        private String byFieldName;
+        private String overFieldName;
+        private String partitionFieldName;
+        private boolean useNull = false;
+        private ExcludeFrequent excludeFrequent;
+        private List<DetectionRule> rules = Collections.emptyList();
+        // negative means unknown
+        private int detectorIndex = -1;
+
+        public Builder() {
+        }
+
+        public Builder(Detector detector) {
+            detectorDescription = detector.detectorDescription;
+            function = detector.function;
+            fieldName = detector.fieldName;
+            byFieldName = detector.byFieldName;
+            overFieldName = detector.overFieldName;
+            partitionFieldName = detector.partitionFieldName;
+            useNull = detector.useNull;
+            excludeFrequent = detector.excludeFrequent;
+            rules = new ArrayList<>(detector.rules);
+            detectorIndex = detector.detectorIndex;
+        }
+
+        public Builder(String function, String fieldName) {
+            this(DetectorFunction.fromString(function), fieldName);
+        }
+
+        public Builder(DetectorFunction function, String fieldName) {
+            this.function = function;
+            this.fieldName = fieldName;
+        }
+
+        public Builder setDetectorDescription(String detectorDescription) {
+            this.detectorDescription = detectorDescription;
+            return this;
+        }
+
+        public Builder setFunction(String function) {
+            this.function = DetectorFunction.fromString(function);
+            return this;
+        }
+
+        public Builder setFieldName(String fieldName) {
+            this.fieldName = fieldName;
+            return this;
+        }
+
+        public Builder setByFieldName(String byFieldName) {
+            this.byFieldName = byFieldName;
+            return this;
+        }
+
+        public Builder setOverFieldName(String overFieldName) {
+            this.overFieldName = overFieldName;
+            return this;
+        }
+
+        public Builder setPartitionFieldName(String partitionFieldName) {
+            this.partitionFieldName = partitionFieldName;
+            return this;
+        }
+
+        public Builder setUseNull(boolean useNull) {
+            this.useNull = useNull;
+            return this;
+        }
+
+        public Builder setExcludeFrequent(ExcludeFrequent excludeFrequent) {
+            this.excludeFrequent = excludeFrequent;
+            return this;
+        }
+
+        public Builder setRules(List<DetectionRule> rules) {
+            this.rules = rules;
+            return this;
+        }
+
+        public Builder setDetectorIndex(int detectorIndex) {
+            this.detectorIndex = detectorIndex;
+            return this;
+        }
+
+        public Detector build() {
+            return new Detector(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName,
+                useNull, excludeFrequent, rules, detectorIndex);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorFunction.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorFunction.java
new file mode 100644
index 0000000000000..5d9a06948d0fb
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorFunction.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Locale;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public enum DetectorFunction {
+
+    COUNT,
+    LOW_COUNT,
+    HIGH_COUNT,
+    NON_ZERO_COUNT("nzc"),
+    LOW_NON_ZERO_COUNT("low_nzc"),
+    HIGH_NON_ZERO_COUNT("high_nzc"),
+    DISTINCT_COUNT("dc"),
+    LOW_DISTINCT_COUNT("low_dc"),
+    HIGH_DISTINCT_COUNT("high_dc"),
+    RARE,
+    FREQ_RARE,
+    INFO_CONTENT,
+    LOW_INFO_CONTENT,
+    HIGH_INFO_CONTENT,
+    METRIC,
+    MEAN,
+    LOW_MEAN,
+    HIGH_MEAN,
+    AVG,
+    LOW_AVG,
+    HIGH_AVG,
+    MEDIAN,
+    LOW_MEDIAN,
+    HIGH_MEDIAN,
+    MIN,
+    MAX,
+    SUM,
+    LOW_SUM,
+    HIGH_SUM,
+    NON_NULL_SUM,
+    LOW_NON_NULL_SUM,
+    HIGH_NON_NULL_SUM,
+    VARP,
+    LOW_VARP,
+    HIGH_VARP,
+    TIME_OF_DAY,
+    TIME_OF_WEEK,
+    LAT_LONG;
+
+    private Set<String> shortcuts;
+
+    DetectorFunction() {
+        shortcuts = Collections.emptySet();
+    }
+
+    DetectorFunction(String... shortcuts) {
+        this.shortcuts = Arrays.stream(shortcuts).collect(Collectors.toSet());
+    }
+
+    public String getFullName() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+
+    @Override
+    public String toString() {
+        return getFullName();
+    }
+
+    public static DetectorFunction fromString(String op) {
+        for (DetectorFunction function : values()) {
+            if (function.getFullName().equals(op) || function.shortcuts.contains(op)) {
+                return function;
+            }
+        }
+        throw new IllegalArgumentException("Unknown detector function [" + op + "]");
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRef.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRef.java
new file mode 100644
index 0000000000000..9afbdf4876fd8
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRef.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+public class FilterRef implements ToXContentObject {
+
+    public static final ParseField FILTER_REF_FIELD = new ParseField("filter_ref");
+    public static final ParseField FILTER_ID = new ParseField("filter_id");
+    public static final ParseField FILTER_TYPE = new ParseField("filter_type");
+
+    public enum FilterType {
+        INCLUDE, EXCLUDE;
+
+        public static FilterType fromString(String value) {
+            return valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    public static final ConstructingObjectParser<FilterRef, Void> PARSER =
+        new ConstructingObjectParser<>(FILTER_REF_FIELD.getPreferredName(), true, a -> new FilterRef((String) a[0], (FilterType) a[1]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), FILTER_ID);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return FilterType.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, FILTER_TYPE, ObjectParser.ValueType.STRING);
+    }
+
+    private final String filterId;
+    private final FilterType filterType;
+
+    public FilterRef(String filterId, FilterType filterType) {
+        this.filterId = Objects.requireNonNull(filterId);
+        this.filterType = filterType == null ? FilterType.INCLUDE : filterType;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(FILTER_ID.getPreferredName(), filterId);
+        builder.field(FILTER_TYPE.getPreferredName(), filterType);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj instanceof FilterRef == false) {
+            return false;
+        }
+
+        FilterRef other = (FilterRef) obj;
+        return Objects.equals(filterId, other.filterId) && Objects.equals(filterType, other.filterType);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(filterId, filterType);
+    }
+
+    public String getFilterId() {
+        return filterId;
+    }
+
+    public FilterType getFilterType() {
+        return filterType;
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java
new file mode 100644
index 0000000000000..6bc1be3b56384
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java
@@ -0,0 +1,592 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * This class represents a configured and created Job. The creation time is set
+ * to the time the object was constructed and the finished time and last
+ * data time fields are {@code null} until the job has seen some data or it is
+ * finished respectively.
+ */
+public class Job implements ToXContentObject {
+
+    public static final String ANOMALY_DETECTOR_JOB_TYPE = "anomaly_detector";
+
+    /*
+     * Field names used in serialization
+     */
+    public static final ParseField ID = new ParseField("job_id");
+    public static final ParseField JOB_TYPE = new ParseField("job_type");
+    public static final ParseField GROUPS = new ParseField("groups");
+    public static final ParseField ANALYSIS_CONFIG = AnalysisConfig.ANALYSIS_CONFIG;
+    public static final ParseField ANALYSIS_LIMITS = new ParseField("analysis_limits");
+    public static final ParseField CREATE_TIME = new ParseField("create_time");
+    public static final ParseField CUSTOM_SETTINGS = new ParseField("custom_settings");
+    public static final ParseField DATA_DESCRIPTION = new ParseField("data_description");
+    public static final ParseField DESCRIPTION = new ParseField("description");
+    public static final ParseField FINISHED_TIME = new ParseField("finished_time");
+    public static final ParseField LAST_DATA_TIME = new ParseField("last_data_time");
+    public static final ParseField ESTABLISHED_MODEL_MEMORY = new ParseField("established_model_memory");
+    public static final ParseField MODEL_PLOT_CONFIG = new ParseField("model_plot_config");
+    public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalization_window_days");
+    public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("background_persist_interval");
+    public static final ParseField MODEL_SNAPSHOT_RETENTION_DAYS = new ParseField("model_snapshot_retention_days");
+    public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days");
+    public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id");
+    public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name");
+    public static final ParseField DELETED = new ParseField("deleted");
+
+    public static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>("job_details", true, Builder::new);
+
+    static {
+        PARSER.declareString(Builder::setId, ID);
+        PARSER.declareString(Builder::setJobType, JOB_TYPE);
+        PARSER.declareStringArray(Builder::setGroups, GROUPS);
+        PARSER.declareStringOrNull(Builder::setDescription, DESCRIPTION);
+        PARSER.declareField(Builder::setCreateTime,
+            (p) -> TimeUtil.parseTimeField(p, CREATE_TIME.getPreferredName()),
+            CREATE_TIME,
+            ValueType.VALUE);
+        PARSER.declareField(Builder::setFinishedTime,
+            (p) -> TimeUtil.parseTimeField(p, FINISHED_TIME.getPreferredName()),
+            FINISHED_TIME,
+            ValueType.VALUE);
+        PARSER.declareField(Builder::setLastDataTime,
+            (p) -> TimeUtil.parseTimeField(p, LAST_DATA_TIME.getPreferredName()),
+            LAST_DATA_TIME,
+            ValueType.VALUE);
+        PARSER.declareLong(Builder::setEstablishedModelMemory, ESTABLISHED_MODEL_MEMORY);
+        PARSER.declareObject(Builder::setAnalysisConfig, AnalysisConfig.PARSER, ANALYSIS_CONFIG);
+        PARSER.declareObject(Builder::setAnalysisLimits, AnalysisLimits.PARSER, ANALYSIS_LIMITS);
+        PARSER.declareObject(Builder::setDataDescription, DataDescription.PARSER, DATA_DESCRIPTION);
+        PARSER.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.PARSER, MODEL_PLOT_CONFIG);
+        PARSER.declareLong(Builder::setRenormalizationWindowDays, RENORMALIZATION_WINDOW_DAYS);
+        PARSER.declareString((builder, val) -> builder.setBackgroundPersistInterval(
+            TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL);
+        PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS);
+        PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS);
+        PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.map(), CUSTOM_SETTINGS, ValueType.OBJECT);
+        PARSER.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID);
+        PARSER.declareString(Builder::setResultsIndexName, RESULTS_INDEX_NAME);
+        PARSER.declareBoolean(Builder::setDeleted, DELETED);
+    }
+
+    private final String jobId;
+    private final String jobType;
+
+    private final List<String> groups;
+    private final String description;
+    private final Date createTime;
+    private final Date finishedTime;
+    private final Date lastDataTime;
+    private final Long establishedModelMemory;
+    private final AnalysisConfig analysisConfig;
+    private final AnalysisLimits analysisLimits;
+    private final DataDescription dataDescription;
+    private final ModelPlotConfig modelPlotConfig;
+    private final Long renormalizationWindowDays;
+    private final TimeValue backgroundPersistInterval;
+    private final Long modelSnapshotRetentionDays;
+    private final Long resultsRetentionDays;
+    private final Map<String, Object> customSettings;
+    private final String modelSnapshotId;
+    private final String resultsIndexName;
+    private final boolean deleted;
+
+    private Job(String jobId, String jobType, List<String> groups, String description, Date createTime,
+                Date finishedTime, Date lastDataTime, Long establishedModelMemory,
+                AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription,
+                ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval,
+                Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map<String, Object> customSettings,
+                String modelSnapshotId, String resultsIndexName, boolean deleted) {
+
+        this.jobId = jobId;
+        this.jobType = jobType;
+        this.groups = Collections.unmodifiableList(groups);
+        this.description = description;
+        this.createTime = createTime;
+        this.finishedTime = finishedTime;
+        this.lastDataTime = lastDataTime;
+        this.establishedModelMemory = establishedModelMemory;
+        this.analysisConfig = analysisConfig;
+        this.analysisLimits = analysisLimits;
+        this.dataDescription = dataDescription;
+        this.modelPlotConfig = modelPlotConfig;
+        this.renormalizationWindowDays = renormalizationWindowDays;
+        this.backgroundPersistInterval = backgroundPersistInterval;
+        this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
+        this.resultsRetentionDays = resultsRetentionDays;
+        this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings);
+        this.modelSnapshotId = modelSnapshotId;
+        this.resultsIndexName = resultsIndexName;
+        this.deleted = deleted;
+    }
+
+    /**
+     * Return the Job Id.
+     *
+     * @return The job Id string
+     */
+    public String getId() {
+        return jobId;
+    }
+
+    public String getJobType() {
+        return jobType;
+    }
+
+    public List<String> getGroups() {
+        return groups;
+    }
+
+    /**
+     * Private version of getResultsIndexName so that a job can be built from another
+     * job and pass index name validation
+     *
+     * @return The job's index name, minus prefix
+     */
+    private String getResultsIndexNameNoPrefix() {
+        return resultsIndexName;
+    }
+
+    /**
+     * The job description
+     *
+     * @return job description
+     */
+    public String getDescription() {
+        return description;
+    }
+
+    /**
+     * The Job creation time. This name is preferred when serialising to the
+     * REST API.
+     *
+     * @return The date the job was created
+     */
+    public Date getCreateTime() {
+        return createTime;
+    }
+
+    /**
+     * The time the job was finished or <code>null</code> if not finished.
+     *
+     * @return The date the job was last retired or <code>null</code>
+     */
+    public Date getFinishedTime() {
+        return finishedTime;
+    }
+
+    /**
+     * The last time data was uploaded to the job or <code>null</code> if no
+     * data has been seen.
+     *
+     * @return The date at which the last data was processed
+     */
+    public Date getLastDataTime() {
+        return lastDataTime;
+    }
+
+    /**
+     * The established model memory of the job, or <code>null</code> if model
+     * memory has not reached equilibrium yet.
+     *
+     * @return The established model memory of the job
+     */
+    public Long getEstablishedModelMemory() {
+        return establishedModelMemory;
+    }
+
+    /**
+     * The analysis configuration object
+     *
+     * @return The AnalysisConfig
+     */
+    public AnalysisConfig getAnalysisConfig() {
+        return analysisConfig;
+    }
+
+    /**
+     * The analysis options object
+     *
+     * @return The AnalysisLimits
+     */
+    public AnalysisLimits getAnalysisLimits() {
+        return analysisLimits;
+    }
+
+    public ModelPlotConfig getModelPlotConfig() {
+        return modelPlotConfig;
+    }
+
+    /**
+     * If not set the input data is assumed to be csv with a '_time' field in
+     * epoch format.
+     *
+     * @return A DataDescription or <code>null</code>
+     * @see DataDescription
+     */
+    public DataDescription getDataDescription() {
+        return dataDescription;
+    }
+
+    /**
+     * The duration of the renormalization window in days
+     *
+     * @return renormalization window in days
+     */
+    public Long getRenormalizationWindowDays() {
+        return renormalizationWindowDays;
+    }
+
+    /**
+     * The background persistence interval
+     *
+     * @return background persistence interval
+     */
+    public TimeValue getBackgroundPersistInterval() {
+        return backgroundPersistInterval;
+    }
+
+    public Long getModelSnapshotRetentionDays() {
+        return modelSnapshotRetentionDays;
+    }
+
+    public Long getResultsRetentionDays() {
+        return resultsRetentionDays;
+    }
+
+    public Map<String, Object> getCustomSettings() {
+        return customSettings;
+    }
+
+    public String getModelSnapshotId() {
+        return modelSnapshotId;
+    }
+
+    public boolean isDeleted() {
+        return deleted;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        final String humanReadableSuffix = "_string";
+
+        builder.field(ID.getPreferredName(), jobId);
+        builder.field(JOB_TYPE.getPreferredName(), jobType);
+
+        if (groups.isEmpty() == false) {
+            builder.field(GROUPS.getPreferredName(), groups);
+        }
+        if (description != null) {
+            builder.field(DESCRIPTION.getPreferredName(), description);
+        }
+        if (createTime != null) {
+            builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime());
+        }
+        if (finishedTime != null) {
+            builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix,
+                finishedTime.getTime());
+        }
+        if (lastDataTime != null) {
+            builder.timeField(LAST_DATA_TIME.getPreferredName(), LAST_DATA_TIME.getPreferredName() + humanReadableSuffix,
+                lastDataTime.getTime());
+        }
+        if (establishedModelMemory != null) {
+            builder.field(ESTABLISHED_MODEL_MEMORY.getPreferredName(), establishedModelMemory);
+        }
+        builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params);
+        if (analysisLimits != null) {
+            builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params);
+        }
+        if (dataDescription != null) {
+            builder.field(DATA_DESCRIPTION.getPreferredName(), dataDescription, params);
+        }
+        if (modelPlotConfig != null) {
+            builder.field(MODEL_PLOT_CONFIG.getPreferredName(), modelPlotConfig, params);
+        }
+        if (renormalizationWindowDays != null) {
+            builder.field(RENORMALIZATION_WINDOW_DAYS.getPreferredName(), renormalizationWindowDays);
+        }
+        if (backgroundPersistInterval != null) {
+            builder.field(BACKGROUND_PERSIST_INTERVAL.getPreferredName(), backgroundPersistInterval.getStringRep());
+        }
+        if (modelSnapshotRetentionDays != null) {
+            builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays);
+        }
+        if (resultsRetentionDays != null) {
+            builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
+        }
+        if (customSettings != null) {
+            builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings);
+        }
+        if (modelSnapshotId != null) {
+            builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId);
+        }
+        if (resultsIndexName != null) {
+            builder.field(RESULTS_INDEX_NAME.getPreferredName(), resultsIndexName);
+        }
+        if (params.paramAsBoolean("all", false)) {
+            builder.field(DELETED.getPreferredName(), deleted);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        Job that = (Job) other;
+        return Objects.equals(this.jobId, that.jobId)
+            && Objects.equals(this.jobType, that.jobType)
+            && Objects.equals(this.groups, that.groups)
+            && Objects.equals(this.description, that.description)
+            && Objects.equals(this.createTime, that.createTime)
+            && Objects.equals(this.finishedTime, that.finishedTime)
+            && Objects.equals(this.lastDataTime, that.lastDataTime)
+            && Objects.equals(this.establishedModelMemory, that.establishedModelMemory)
+            && Objects.equals(this.analysisConfig, that.analysisConfig)
+            && Objects.equals(this.analysisLimits, that.analysisLimits)
+            && Objects.equals(this.dataDescription, that.dataDescription)
+            && Objects.equals(this.modelPlotConfig, that.modelPlotConfig)
+            && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays)
+            && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval)
+            && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
+            && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
+            && Objects.equals(this.customSettings, that.customSettings)
+            && Objects.equals(this.modelSnapshotId, that.modelSnapshotId)
+            && Objects.equals(this.resultsIndexName, that.resultsIndexName)
+            && Objects.equals(this.deleted, that.deleted);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime, lastDataTime, establishedModelMemory,
+            analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
+            backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
+            modelSnapshotId, resultsIndexName, deleted);
+    }
+
+    @Override
+    public final String toString() {
+        return Strings.toString(this);
+    }
+
+    public static class Builder {
+
+        private String id;
+        private String jobType = ANOMALY_DETECTOR_JOB_TYPE;
+        private List<String> groups = Collections.emptyList();
+        private String description;
+        private AnalysisConfig analysisConfig;
+        private AnalysisLimits analysisLimits;
+        private DataDescription dataDescription;
+        private Date createTime;
+        private Date finishedTime;
+        private Date lastDataTime;
+        private Long establishedModelMemory;
+        private ModelPlotConfig modelPlotConfig;
+        private Long renormalizationWindowDays;
+        private TimeValue backgroundPersistInterval;
+        private Long modelSnapshotRetentionDays;
+        private Long resultsRetentionDays;
+        private Map<String, Object> customSettings;
+        private String modelSnapshotId;
+        private String resultsIndexName;
+        private boolean deleted;
+
+        public Builder() {
+        }
+
+        public Builder(String id) {
+            this.id = id;
+        }
+
+        public Builder(Job job) {
+            this.id = job.getId();
+            this.jobType = job.getJobType();
+            this.groups = job.getGroups();
+            this.description = job.getDescription();
+            this.analysisConfig = job.getAnalysisConfig();
+            this.analysisLimits = job.getAnalysisLimits();
+            this.dataDescription = job.getDataDescription();
+            this.createTime = job.getCreateTime();
+            this.finishedTime = job.getFinishedTime();
+            this.lastDataTime = job.getLastDataTime();
+            this.establishedModelMemory = job.getEstablishedModelMemory();
+            this.modelPlotConfig = job.getModelPlotConfig();
+            this.renormalizationWindowDays = job.getRenormalizationWindowDays();
+            this.backgroundPersistInterval = job.getBackgroundPersistInterval();
+            this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays();
+            this.resultsRetentionDays = job.getResultsRetentionDays();
+            this.customSettings = job.getCustomSettings();
+            this.modelSnapshotId = job.getModelSnapshotId();
+            this.resultsIndexName = job.getResultsIndexNameNoPrefix();
+            this.deleted = job.isDeleted();
+        }
+
+        public Builder setId(String id) {
+            this.id = id;
+            return this;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public Builder setJobType(String jobType) {
+            this.jobType = jobType;
+            return this;
+        }
+
+        public Builder setGroups(List<String> groups) {
+            this.groups = groups == null ? Collections.emptyList() : groups;
+            return this;
+        }
+
+        public Builder setCustomSettings(Map<String, Object> customSettings) {
+            this.customSettings = customSettings;
+            return this;
+        }
+
+        public Builder setDescription(String description) {
+            this.description = description;
+            return this;
+        }
+
+        public Builder setAnalysisConfig(AnalysisConfig.Builder configBuilder) {
+            analysisConfig = Objects.requireNonNull(configBuilder, ANALYSIS_CONFIG.getPreferredName()).build();
+            return this;
+        }
+
+        public Builder setAnalysisLimits(AnalysisLimits analysisLimits) {
+            this.analysisLimits = Objects.requireNonNull(analysisLimits, ANALYSIS_LIMITS.getPreferredName());
+            return this;
+        }
+
+        Builder setCreateTime(Date createTime) {
+            this.createTime = createTime;
+            return this;
+        }
+
+        Builder setFinishedTime(Date finishedTime) {
+            this.finishedTime = finishedTime;
+            return this;
+        }
+
+        /**
+         * Set the wall clock time of the last data upload
+         *
+         * @param lastDataTime Wall clock time
+         */
+        public Builder setLastDataTime(Date lastDataTime) {
+            this.lastDataTime = lastDataTime;
+            return this;
+        }
+
+        public Builder setEstablishedModelMemory(Long establishedModelMemory) {
+            this.establishedModelMemory = establishedModelMemory;
+            return this;
+        }
+
+        public Builder setDataDescription(DataDescription.Builder description) {
+            dataDescription = Objects.requireNonNull(description, DATA_DESCRIPTION.getPreferredName()).build();
+            return this;
+        }
+
+        public Builder setModelPlotConfig(ModelPlotConfig modelPlotConfig) {
+            this.modelPlotConfig = modelPlotConfig;
+            return this;
+        }
+
+        public Builder setBackgroundPersistInterval(TimeValue backgroundPersistInterval) {
+            this.backgroundPersistInterval = backgroundPersistInterval;
+            return this;
+        }
+
+        public Builder setRenormalizationWindowDays(Long renormalizationWindowDays) {
+            this.renormalizationWindowDays = renormalizationWindowDays;
+            return this;
+        }
+
+        public Builder setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays) {
+            this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
+            return this;
+        }
+
+        public Builder setResultsRetentionDays(Long resultsRetentionDays) {
+            this.resultsRetentionDays = resultsRetentionDays;
+            return this;
+        }
+
+        public Builder setModelSnapshotId(String modelSnapshotId) {
+            this.modelSnapshotId = modelSnapshotId;
+            return this;
+        }
+
+        public Builder setResultsIndexName(String resultsIndexName) {
+            this.resultsIndexName = resultsIndexName;
+            return this;
+        }
+
+        public Builder setDeleted(boolean deleted) {
+            this.deleted = deleted;
+            return this;
+        }
+
+        /**
+         * Builds a job.
+         *
+         * @return The job
+         */
+        public Job build() {
+            Objects.requireNonNull(id,  "[" + ID.getPreferredName() + "] must not be null");
+            Objects.requireNonNull(jobType,  "[" + JOB_TYPE.getPreferredName() + "] must not be null");
+            return new Job(
+                id, jobType, groups, description, createTime, finishedTime, lastDataTime, establishedModelMemory,
+                analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
+                backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
+                modelSnapshotId, resultsIndexName, deleted);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilter.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilter.java
new file mode 100644
index 0000000000000..bcbc0c295c2d5
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilter.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+public class MlFilter implements ToXContentObject {
+
+    public static final ParseField TYPE = new ParseField("type");
+    public static final ParseField ID = new ParseField("filter_id");
+    public static final ParseField DESCRIPTION = new ParseField("description");
+    public static final ParseField ITEMS = new ParseField("items");
+
+    // For QueryPage
+    public static final ParseField RESULTS_FIELD = new ParseField("filters");
+
+    public static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>(TYPE.getPreferredName(), true, Builder::new);
+
+    static {
+        PARSER.declareString((builder, s) -> {}, TYPE);
+        PARSER.declareString(Builder::setId, ID);
+        PARSER.declareStringOrNull(Builder::setDescription, DESCRIPTION);
+        PARSER.declareStringArray(Builder::setItems, ITEMS);
+    }
+
+    private final String id;
+    private final String description;
+    private final SortedSet<String> items;
+
+    private MlFilter(String id, String description, SortedSet<String> items) {
+        this.id = Objects.requireNonNull(id);
+        this.description = description;
+        this.items = Collections.unmodifiableSortedSet(items);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ID.getPreferredName(), id);
+        if (description != null) {
+            builder.field(DESCRIPTION.getPreferredName(), description);
+        }
+        builder.field(ITEMS.getPreferredName(), items);
+        // Don't include TYPE as it's fixed
+        builder.endObject();
+        return builder;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public SortedSet<String> getItems() {
+        return items;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == this) {
+            return true;
+        }
+
+        if (obj instanceof MlFilter == false) {
+            return false;
+        }
+
+        MlFilter other = (MlFilter) obj;
+        return id.equals(other.id) && Objects.equals(description, other.description) && items.equals(other.items);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, description, items);
+    }
+
+    public static Builder builder(String filterId) {
+        return new Builder().setId(filterId);
+    }
+
+    public static class Builder {
+
+        private String id;
+        private String description;
+        private SortedSet<String> items = new TreeSet<>();
+
+        private Builder() {
+        }
+
+        public Builder setId(String id) {
+            this.id = Objects.requireNonNull(id);
+            return this;
+        }
+
+        @Nullable
+        public String getId() {
+            return id;
+        }
+
+        public Builder setDescription(String description) {
+            this.description = description;
+            return this;
+        }
+
+        public Builder setItems(SortedSet<String> items) {
+            this.items = Objects.requireNonNull(items);
+            return this;
+        }
+
+        public Builder setItems(List<String> items) {
+            this.items = new TreeSet<>(items);
+            return this;
+        }
+
+        public Builder setItems(String... items) {
+            setItems(Arrays.asList(items));
+            return this;
+        }
+
+        public MlFilter build() {
+            return new MlFilter(id, description, items);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/ModelPlotConfig.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/ModelPlotConfig.java
new file mode 100644
index 0000000000000..59b0252a7660e
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/ModelPlotConfig.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ModelPlotConfig implements ToXContentObject {
+
+    private static final ParseField TYPE_FIELD = new ParseField("model_plot_config");
+    private static final ParseField ENABLED_FIELD = new ParseField("enabled");
+    public static final ParseField TERMS_FIELD = new ParseField("terms");
+
+    public static final ConstructingObjectParser<ModelPlotConfig, Void> PARSER =
+        new ConstructingObjectParser<>(TYPE_FIELD.getPreferredName(), true, a -> new ModelPlotConfig((boolean) a[0], (String) a[1]));
+
+    static {
+        PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TERMS_FIELD);
+    }
+
+    private final boolean enabled;
+    private final String terms;
+
+    public ModelPlotConfig(boolean enabled, String terms) {
+        this.enabled = enabled;
+        this.terms = terms;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ENABLED_FIELD.getPreferredName(), enabled);
+        if (terms != null) {
+            builder.field(TERMS_FIELD.getPreferredName(), terms);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    public String getTerms() {
+        return this.terms;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other instanceof ModelPlotConfig == false) {
+            return false;
+        }
+
+        ModelPlotConfig that = (ModelPlotConfig) other;
+        return this.enabled == that.enabled && Objects.equals(this.terms, that.terms);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(enabled, terms);
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Operator.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Operator.java
new file mode 100644
index 0000000000000..c3dc52e5a3cb9
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Operator.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.ParseField;
+
+import java.util.Locale;
+
+/**
+ * Enum representing logical comparisons on doubles
+ */
+public enum Operator {
+    GT {
+        @Override
+        public boolean test(double lhs, double rhs) {
+            return Double.compare(lhs, rhs) > 0;
+        }
+    },
+    GTE {
+        @Override
+        public boolean test(double lhs, double rhs) {
+            return Double.compare(lhs, rhs) >= 0;
+        }
+    },
+    LT {
+        @Override
+        public boolean test(double lhs, double rhs) {
+            return Double.compare(lhs, rhs) < 0;
+        }
+    },
+    LTE {
+        @Override
+        public boolean test(double lhs, double rhs) {
+            return Double.compare(lhs, rhs) <= 0;
+        }
+    };
+    // EQ was considered but given the oddity of such a
+    // condition and the fact that it would be a numerically
+    // unstable condition, it was rejected.
+
+    public static final ParseField OPERATOR_FIELD = new ParseField("operator");
+
+    public boolean test(double lhs, double rhs) {
+        return false;
+    }
+
+    public static Operator fromString(String name) {
+        return valueOf(name.trim().toUpperCase(Locale.ROOT));
+    }
+
+    @Override
+    public String toString() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateFilter.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleAction.java
similarity index 60%
rename from server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateFilter.java
rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleAction.java
index 6ed9664c92b1c..9e2364b4fd960 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateFilter.java
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleAction.java
@@ -16,19 +16,26 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+package org.elasticsearch.protocol.xpack.ml.job.config;
 
-package org.elasticsearch.cluster.metadata;
+import java.util.Locale;
 
-import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
-
-/**
- * Enables filtering the index templates that will be applied for an index, per create index request.
- */
-public interface IndexTemplateFilter {
+public enum RuleAction {
+    SKIP_RESULT,
+    SKIP_MODEL_UPDATE;
 
     /**
-     * @return  {@code true} if the given template should be applied on the newly created index,
-     *          {@code false} otherwise.
+     * Case-insensitive from string method.
+     *
+     * @param value String representation
+     * @return The rule action
      */
-    boolean apply(CreateIndexClusterStateUpdateRequest request, IndexTemplateMetaData template);
+    public static RuleAction fromString(String value) {
+        return RuleAction.valueOf(value.toUpperCase(Locale.ROOT));
+    }
+
+    @Override
+    public String toString() {
+        return name().toLowerCase(Locale.ROOT);
+    }
 }
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleCondition.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleCondition.java
new file mode 100644
index 0000000000000..ec19547fe13be
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleCondition.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+public class RuleCondition implements ToXContentObject {
+
+    public static final ParseField RULE_CONDITION_FIELD = new ParseField("rule_condition");
+
+    public static final ParseField APPLIES_TO_FIELD = new ParseField("applies_to");
+    public static final ParseField VALUE_FIELD = new ParseField("value");
+
+    public static final ConstructingObjectParser<RuleCondition, Void> PARSER =
+        new ConstructingObjectParser<>(RULE_CONDITION_FIELD.getPreferredName(), true,
+            a -> new RuleCondition((AppliesTo) a[0], (Operator) a[1], (double) a[2]));
+
+    static {
+        PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return AppliesTo.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, APPLIES_TO_FIELD, ValueType.STRING);
+        PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return Operator.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, Operator.OPERATOR_FIELD, ValueType.STRING);
+        PARSER.declareDouble(ConstructingObjectParser.constructorArg(), VALUE_FIELD);
+    }
+
+    private final AppliesTo appliesTo;
+    private final Operator operator;
+    private final double value;
+
+    public RuleCondition(AppliesTo appliesTo, Operator operator, double value) {
+        this.appliesTo = appliesTo;
+        this.operator = operator;
+        this.value = value;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(APPLIES_TO_FIELD.getPreferredName(), appliesTo);
+        builder.field(Operator.OPERATOR_FIELD.getPreferredName(), operator);
+        builder.field(VALUE_FIELD.getPreferredName(), value);
+        builder.endObject();
+        return builder;
+    }
+
+    public AppliesTo getAppliesTo() {
+        return appliesTo;
+    }
+
+    public Operator getOperator() {
+        return operator;
+    }
+
+    public double getValue() {
+        return value;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj instanceof RuleCondition == false) {
+            return false;
+        }
+
+        RuleCondition other = (RuleCondition) obj;
+        return appliesTo == other.appliesTo && operator == other.operator && value == other.value;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(appliesTo, operator, value);
+    }
+
+    public static RuleCondition createTime(Operator operator, long epochSeconds) {
+        return new RuleCondition(AppliesTo.TIME, operator, epochSeconds);
+    }
+
+    public enum AppliesTo {
+        ACTUAL,
+        TYPICAL,
+        DIFF_FROM_TYPICAL,
+        TIME;
+
+        public static AppliesTo fromString(String value) {
+            return valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScope.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScope.java
new file mode 100644
index 0000000000000..aa12d5ea2a2bd
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScope.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ContextParser;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class RuleScope implements ToXContentObject {
+
+    public static ContextParser<Void, RuleScope> parser() {
+        return (p, c) -> {
+            Map<String, Object> unparsedScope = p.map();
+            if (unparsedScope.isEmpty()) {
+                return new RuleScope();
+            }
+            Map<String, FilterRef> scope = new HashMap<>();
+            for (Map.Entry<String, Object> entry : unparsedScope.entrySet()) {
+                try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+                    @SuppressWarnings("unchecked")
+                    Map<String, ?> value = (Map<String, ?>) entry.getValue();
+                    builder.map(value);
+                    try (XContentParser scopeParser = XContentFactory.xContent(builder.contentType()).createParser(
+                            NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, Strings.toString(builder))) {
+                        scope.put(entry.getKey(), FilterRef.PARSER.parse(scopeParser, null));
+                    }
+                }
+            }
+            return new RuleScope(scope);
+        };
+    }
+
+    private final Map<String, FilterRef> scope;
+
+    public RuleScope() {
+        scope = Collections.emptyMap();
+    }
+
+    public RuleScope(Map<String, FilterRef> scope) {
+        this.scope = Collections.unmodifiableMap(scope);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.map(scope);
+    }
+
+    public boolean isEmpty() {
+        return scope.isEmpty();
+    }
+
+    public Set<String> getReferencedFilters() {
+        return scope.values().stream().map(FilterRef::getFilterId).collect(Collectors.toSet());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj instanceof RuleScope == false) {
+            return false;
+        }
+
+        RuleScope other = (RuleScope) obj;
+        return Objects.equals(scope, other.scope);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(scope);
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+
+        private Map<String, FilterRef> scope = new HashMap<>();
+
+        public Builder() {
+        }
+
+        public Builder(RuleScope otherScope) {
+            scope = new HashMap<>(otherScope.scope);
+        }
+
+        public Builder exclude(String field, String filterId) {
+            scope.put(field, new FilterRef(filterId, FilterRef.FilterType.EXCLUDE));
+            return this;
+        }
+
+        public Builder include(String field, String filterId) {
+            scope.put(field, new FilterRef(filterId, FilterRef.FilterType.INCLUDE));
+            return this;
+        }
+
+        public RuleScope build() {
+            return new RuleScope(scope);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCounts.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCounts.java
new file mode 100644
index 0000000000000..e07312d12e1f4
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCounts.java
@@ -0,0 +1,415 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.process;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.Objects;
+
+/**
+ * Job processed record counts.
+ * <p>
+ * The getInput... methods return the actual number of
+ * fields/records sent the the API including invalid records.
+ * The getProcessed... methods are the number sent to the
+ * Engine.
+ * <p>
+ * The <code>inputRecordCount</code> field is calculated so it
+ * should not be set in deserialization but it should be serialised
+ * so the field is visible.
+ */
+public class DataCounts implements ToXContentObject {
+
+    public static final ParseField PROCESSED_RECORD_COUNT = new ParseField("processed_record_count");
+    public static final ParseField PROCESSED_FIELD_COUNT = new ParseField("processed_field_count");
+    public static final ParseField INPUT_BYTES = new ParseField("input_bytes");
+    public static final ParseField INPUT_RECORD_COUNT = new ParseField("input_record_count");
+    public static final ParseField INPUT_FIELD_COUNT = new ParseField("input_field_count");
+    public static final ParseField INVALID_DATE_COUNT = new ParseField("invalid_date_count");
+    public static final ParseField MISSING_FIELD_COUNT = new ParseField("missing_field_count");
+    public static final ParseField OUT_OF_ORDER_TIME_COUNT = new ParseField("out_of_order_timestamp_count");
+    public static final ParseField EMPTY_BUCKET_COUNT = new ParseField("empty_bucket_count");
+    public static final ParseField SPARSE_BUCKET_COUNT = new ParseField("sparse_bucket_count");
+    public static final ParseField BUCKET_COUNT = new ParseField("bucket_count");
+    public static final ParseField EARLIEST_RECORD_TIME = new ParseField("earliest_record_timestamp");
+    public static final ParseField LATEST_RECORD_TIME = new ParseField("latest_record_timestamp");
+    public static final ParseField LAST_DATA_TIME = new ParseField("last_data_time");
+    public static final ParseField LATEST_EMPTY_BUCKET_TIME = new ParseField("latest_empty_bucket_timestamp");
+    public static final ParseField LATEST_SPARSE_BUCKET_TIME = new ParseField("latest_sparse_bucket_timestamp");
+
+    public static final ConstructingObjectParser<DataCounts, Void> PARSER = new ConstructingObjectParser<>("data_counts", true,
+            a -> new DataCounts((String) a[0], (long) a[1], (long) a[2], (long) a[3], (long) a[4], (long) a[5], (long) a[6],
+                    (long) a[7], (long) a[8], (long) a[9], (long) a[10], (Date) a[11], (Date) a[12], (Date) a[13], (Date) a[14],
+                    (Date) a[15]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), PROCESSED_RECORD_COUNT);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), PROCESSED_FIELD_COUNT);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), INPUT_BYTES);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), INPUT_FIELD_COUNT);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), INVALID_DATE_COUNT);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), MISSING_FIELD_COUNT);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), OUT_OF_ORDER_TIME_COUNT);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), EMPTY_BUCKET_COUNT);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), SPARSE_BUCKET_COUNT);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_COUNT);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
+            (p) -> TimeUtil.parseTimeField(p, EARLIEST_RECORD_TIME.getPreferredName()),
+            EARLIEST_RECORD_TIME,
+            ValueType.VALUE);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
+            (p) -> TimeUtil.parseTimeField(p, LATEST_RECORD_TIME.getPreferredName()),
+            LATEST_RECORD_TIME,
+            ValueType.VALUE);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
+            (p) -> TimeUtil.parseTimeField(p, LAST_DATA_TIME.getPreferredName()),
+            LAST_DATA_TIME,
+            ValueType.VALUE);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
+            (p) -> TimeUtil.parseTimeField(p, LATEST_EMPTY_BUCKET_TIME.getPreferredName()),
+            LATEST_EMPTY_BUCKET_TIME,
+            ValueType.VALUE);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
+            (p) -> TimeUtil.parseTimeField(p, LATEST_SPARSE_BUCKET_TIME.getPreferredName()),
+            LATEST_SPARSE_BUCKET_TIME,
+            ValueType.VALUE);
+    }
+
+    private final String jobId;
+    private long processedRecordCount;
+    private long processedFieldCount;
+    private long inputBytes;
+    private long inputFieldCount;
+    private long invalidDateCount;
+    private long missingFieldCount;
+    private long outOfOrderTimeStampCount;
+    private long emptyBucketCount;
+    private long sparseBucketCount;
+    private long bucketCount;
+    private Date earliestRecordTimeStamp;
+    private Date latestRecordTimeStamp;
+    private Date lastDataTimeStamp;
+    private Date latestEmptyBucketTimeStamp;
+    private Date latestSparseBucketTimeStamp;
+
+    public DataCounts(String jobId, long processedRecordCount, long processedFieldCount, long inputBytes,
+                      long inputFieldCount, long invalidDateCount, long missingFieldCount, long outOfOrderTimeStampCount,
+                      long emptyBucketCount, long sparseBucketCount, long bucketCount,
+                      Date earliestRecordTimeStamp, Date latestRecordTimeStamp, Date lastDataTimeStamp,
+                      Date latestEmptyBucketTimeStamp, Date latestSparseBucketTimeStamp) {
+        this.jobId = jobId;
+        this.processedRecordCount = processedRecordCount;
+        this.processedFieldCount = processedFieldCount;
+        this.inputBytes = inputBytes;
+        this.inputFieldCount = inputFieldCount;
+        this.invalidDateCount = invalidDateCount;
+        this.missingFieldCount = missingFieldCount;
+        this.outOfOrderTimeStampCount = outOfOrderTimeStampCount;
+        this.emptyBucketCount = emptyBucketCount;
+        this.sparseBucketCount = sparseBucketCount;
+        this.bucketCount = bucketCount;
+        this.latestRecordTimeStamp = latestRecordTimeStamp;
+        this.earliestRecordTimeStamp = earliestRecordTimeStamp;
+        this.lastDataTimeStamp = lastDataTimeStamp;
+        this.latestEmptyBucketTimeStamp = latestEmptyBucketTimeStamp;
+        this.latestSparseBucketTimeStamp = latestSparseBucketTimeStamp;
+    }
+
+    DataCounts(String jobId) {
+        this.jobId = jobId;
+    }
+
+    public DataCounts(DataCounts lhs) {
+        jobId = lhs.jobId;
+        processedRecordCount = lhs.processedRecordCount;
+        processedFieldCount = lhs.processedFieldCount;
+        inputBytes = lhs.inputBytes;
+        inputFieldCount = lhs.inputFieldCount;
+        invalidDateCount = lhs.invalidDateCount;
+        missingFieldCount = lhs.missingFieldCount;
+        outOfOrderTimeStampCount = lhs.outOfOrderTimeStampCount;
+        emptyBucketCount = lhs.emptyBucketCount;
+        sparseBucketCount = lhs.sparseBucketCount;
+        bucketCount = lhs.bucketCount;
+        latestRecordTimeStamp = lhs.latestRecordTimeStamp;
+        earliestRecordTimeStamp = lhs.earliestRecordTimeStamp;
+        lastDataTimeStamp = lhs.lastDataTimeStamp;
+        latestEmptyBucketTimeStamp = lhs.latestEmptyBucketTimeStamp;
+        latestSparseBucketTimeStamp = lhs.latestSparseBucketTimeStamp;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    /**
+     * Number of records processed by this job.
+     * This value is the number of records sent passed on to
+     * the engine i.e. {@linkplain #getInputRecordCount()} minus
+     * records with bad dates or out of order
+     *
+     * @return Number of records processed by this job {@code long}
+     */
+    public long getProcessedRecordCount() {
+        return processedRecordCount;
+    }
+
+    /**
+     * Number of data points (processed record count * the number
+     * of analysed fields) processed by this job. This count does
+     * not include the time field.
+     *
+     * @return Number of data points processed by this job {@code long}
+     */
+    public long getProcessedFieldCount() {
+        return processedFieldCount;
+    }
+
+    /**
+     * Total number of input records read.
+     * This = processed record count + date parse error records count
+     * + out of order record count.
+     * <p>
+     * Records with missing fields are counted as they are still written.
+     *
+     * @return Total number of input records read {@code long}
+     */
+    public long getInputRecordCount() {
+        return processedRecordCount + outOfOrderTimeStampCount
+                + invalidDateCount;
+    }
+
+    /**
+     * The total number of bytes sent to this job.
+     * This value includes the bytes from any  records
+     * that have been discarded for any  reason
+     * e.g. because the date cannot be read
+     *
+     * @return Volume in bytes
+     */
+    public long getInputBytes() {
+        return inputBytes;
+    }
+
+    /**
+     * The total number of fields sent to the job
+     * including fields that aren't analysed.
+     *
+     * @return The total number of fields sent to the job
+     */
+    public long getInputFieldCount() {
+        return inputFieldCount;
+    }
+
+    /**
+     * The number of records with an invalid date field that could
+     * not be parsed or converted to epoch time.
+     *
+     * @return The number of records with an invalid date field
+     */
+    public long getInvalidDateCount() {
+        return invalidDateCount;
+    }
+
+    /**
+     * The number of missing fields that had been
+     * configured for analysis.
+     *
+     * @return The number of missing fields
+     */
+    public long getMissingFieldCount() {
+        return missingFieldCount;
+    }
+
+    /**
+     * The number of records with a timestamp that is
+     * before the time of the latest record. Records should
+     * be in ascending chronological order
+     *
+     * @return The number of records with a timestamp that is before the time of the latest record
+     */
+    public long getOutOfOrderTimeStampCount() {
+        return outOfOrderTimeStampCount;
+    }
+
+    /**
+     * The number of buckets with no records in it. Used to measure general data fitness and/or
+     * configuration problems (bucket span).
+     *
+     * @return Number of empty buckets processed by this job {@code long}
+     */
+    public long getEmptyBucketCount() {
+        return emptyBucketCount;
+    }
+
+    /**
+     * The number of buckets with few records compared to the overall counts.
+     * Used to measure general data fitness and/or configuration problems (bucket span).
+     *
+     * @return Number of sparse buckets processed by this job {@code long}
+     */
+    public long getSparseBucketCount() {
+        return sparseBucketCount;
+    }
+
+    /**
+     * The number of buckets overall.
+     *
+     * @return Number of buckets processed by this job {@code long}
+     */
+    public long getBucketCount() {
+        return bucketCount;
+    }
+
+    /**
+     * The time of the first record seen.
+     *
+     * @return The first record time
+     */
+    public Date getEarliestRecordTimeStamp() {
+        return earliestRecordTimeStamp;
+    }
+
+    /**
+     * The time of the latest record seen.
+     *
+     * @return Latest record time
+     */
+    public Date getLatestRecordTimeStamp() {
+        return latestRecordTimeStamp;
+    }
+
+    /**
+     * The wall clock time the latest record was seen.
+     *
+     * @return Wall clock time of the lastest record
+     */
+    public Date getLastDataTimeStamp() {
+        return lastDataTimeStamp;
+    }
+
+    /**
+     * The time of the latest empty bucket seen.
+     *
+     * @return Latest empty bucket time
+     */
+    public Date getLatestEmptyBucketTimeStamp() {
+        return latestEmptyBucketTimeStamp;
+    }
+
+    /**
+     * The time of the latest sparse bucket seen.
+     *
+     * @return Latest sparse bucket time
+     */
+    public Date getLatestSparseBucketTimeStamp() {
+        return latestSparseBucketTimeStamp;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder.field(Job.ID.getPreferredName(), jobId);
+        builder.field(PROCESSED_RECORD_COUNT.getPreferredName(), processedRecordCount);
+        builder.field(PROCESSED_FIELD_COUNT.getPreferredName(), processedFieldCount);
+        builder.field(INPUT_BYTES.getPreferredName(), inputBytes);
+        builder.field(INPUT_FIELD_COUNT.getPreferredName(), inputFieldCount);
+        builder.field(INVALID_DATE_COUNT.getPreferredName(), invalidDateCount);
+        builder.field(MISSING_FIELD_COUNT.getPreferredName(), missingFieldCount);
+        builder.field(OUT_OF_ORDER_TIME_COUNT.getPreferredName(), outOfOrderTimeStampCount);
+        builder.field(EMPTY_BUCKET_COUNT.getPreferredName(), emptyBucketCount);
+        builder.field(SPARSE_BUCKET_COUNT.getPreferredName(), sparseBucketCount);
+        builder.field(BUCKET_COUNT.getPreferredName(), bucketCount);
+        if (earliestRecordTimeStamp != null) {
+            builder.timeField(EARLIEST_RECORD_TIME.getPreferredName(), EARLIEST_RECORD_TIME.getPreferredName() + "_string",
+                earliestRecordTimeStamp.getTime());
+        }
+        if (latestRecordTimeStamp != null) {
+            builder.timeField(LATEST_RECORD_TIME.getPreferredName(), LATEST_RECORD_TIME.getPreferredName() + "_string",
+                latestRecordTimeStamp.getTime());
+        }
+        if (lastDataTimeStamp != null) {
+            builder.timeField(LAST_DATA_TIME.getPreferredName(), LAST_DATA_TIME.getPreferredName() + "_string",
+                lastDataTimeStamp.getTime());
+        }
+        if (latestEmptyBucketTimeStamp != null) {
+            builder.timeField(LATEST_EMPTY_BUCKET_TIME.getPreferredName(), LATEST_EMPTY_BUCKET_TIME.getPreferredName() + "_string",
+                latestEmptyBucketTimeStamp.getTime());
+        }
+        if (latestSparseBucketTimeStamp != null) {
+            builder.timeField(LATEST_SPARSE_BUCKET_TIME.getPreferredName(), LATEST_SPARSE_BUCKET_TIME.getPreferredName() + "_string",
+                latestSparseBucketTimeStamp.getTime());
+        }
+        builder.field(INPUT_RECORD_COUNT.getPreferredName(), getInputRecordCount());
+
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * Equality test
+     */
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        DataCounts that = (DataCounts) other;
+
+        return Objects.equals(this.jobId, that.jobId) &&
+                this.processedRecordCount == that.processedRecordCount &&
+                this.processedFieldCount == that.processedFieldCount &&
+                this.inputBytes == that.inputBytes &&
+                this.inputFieldCount == that.inputFieldCount &&
+                this.invalidDateCount == that.invalidDateCount &&
+                this.missingFieldCount == that.missingFieldCount &&
+                this.outOfOrderTimeStampCount == that.outOfOrderTimeStampCount &&
+                this.emptyBucketCount == that.emptyBucketCount &&
+                this.sparseBucketCount == that.sparseBucketCount &&
+                this.bucketCount == that.bucketCount &&
+                Objects.equals(this.latestRecordTimeStamp, that.latestRecordTimeStamp) &&
+                Objects.equals(this.earliestRecordTimeStamp, that.earliestRecordTimeStamp) &&
+                Objects.equals(this.lastDataTimeStamp, that.lastDataTimeStamp) &&
+                Objects.equals(this.latestEmptyBucketTimeStamp, that.latestEmptyBucketTimeStamp) &&
+                Objects.equals(this.latestSparseBucketTimeStamp, that.latestSparseBucketTimeStamp);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, processedRecordCount, processedFieldCount,
+                inputBytes, inputFieldCount, invalidDateCount, missingFieldCount,
+                outOfOrderTimeStampCount, lastDataTimeStamp, emptyBucketCount, sparseBucketCount, bucketCount,
+                latestRecordTimeStamp, earliestRecordTimeStamp, latestEmptyBucketTimeStamp, latestSparseBucketTimeStamp);
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStats.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStats.java
new file mode 100644
index 0000000000000..50f655b4dd7f1
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStats.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.process;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+import org.elasticsearch.protocol.xpack.ml.job.results.Result;
+import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * Provide access to the C++ model memory usage numbers for the Java process.
+ */
+public class ModelSizeStats implements ToXContentObject {
+
+    /**
+     * Result type
+     */
+    public static final String RESULT_TYPE_VALUE = "model_size_stats";
+    public static final ParseField RESULT_TYPE_FIELD = new ParseField(RESULT_TYPE_VALUE);
+
+    /**
+     * Field Names
+     */
+    public static final ParseField MODEL_BYTES_FIELD = new ParseField("model_bytes");
+    public static final ParseField TOTAL_BY_FIELD_COUNT_FIELD = new ParseField("total_by_field_count");
+    public static final ParseField TOTAL_OVER_FIELD_COUNT_FIELD = new ParseField("total_over_field_count");
+    public static final ParseField TOTAL_PARTITION_FIELD_COUNT_FIELD = new ParseField("total_partition_field_count");
+    public static final ParseField BUCKET_ALLOCATION_FAILURES_COUNT_FIELD = new ParseField("bucket_allocation_failures_count");
+    public static final ParseField MEMORY_STATUS_FIELD = new ParseField("memory_status");
+    public static final ParseField LOG_TIME_FIELD = new ParseField("log_time");
+    public static final ParseField TIMESTAMP_FIELD = new ParseField("timestamp");
+
+    public static final ConstructingObjectParser<Builder, Void> PARSER =
+        new ConstructingObjectParser<>(RESULT_TYPE_VALUE, true, a -> new Builder((String) a[0]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
+        PARSER.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD);
+        PARSER.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD);
+        PARSER.declareLong(Builder::setTotalByFieldCount, TOTAL_BY_FIELD_COUNT_FIELD);
+        PARSER.declareLong(Builder::setTotalOverFieldCount, TOTAL_OVER_FIELD_COUNT_FIELD);
+        PARSER.declareLong(Builder::setTotalPartitionFieldCount, TOTAL_PARTITION_FIELD_COUNT_FIELD);
+        PARSER.declareField(Builder::setLogTime,
+            (p) -> TimeUtil.parseTimeField(p, LOG_TIME_FIELD.getPreferredName()),
+            LOG_TIME_FIELD,
+            ValueType.VALUE);
+        PARSER.declareField(Builder::setTimestamp,
+            (p) -> TimeUtil.parseTimeField(p, TIMESTAMP_FIELD.getPreferredName()),
+            TIMESTAMP_FIELD,
+            ValueType.VALUE);
+        PARSER.declareField(Builder::setMemoryStatus, p -> MemoryStatus.fromString(p.text()), MEMORY_STATUS_FIELD, ValueType.STRING);
+    }
+
+    /**
+     * The status of the memory monitored by the ResourceMonitor. OK is default,
+     * SOFT_LIMIT means that the models have done some aggressive pruning to
+     * keep the memory below the limit, and HARD_LIMIT means that samples have
+     * been dropped
+     */
+    public enum MemoryStatus {
+        OK, SOFT_LIMIT, HARD_LIMIT;
+
+        public static MemoryStatus fromString(String statusName) {
+            return valueOf(statusName.trim().toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    private final String jobId;
+    private final long modelBytes;
+    private final long totalByFieldCount;
+    private final long totalOverFieldCount;
+    private final long totalPartitionFieldCount;
+    private final long bucketAllocationFailuresCount;
+    private final MemoryStatus memoryStatus;
+    private final Date timestamp;
+    private final Date logTime;
+
+    private ModelSizeStats(String jobId, long modelBytes, long totalByFieldCount, long totalOverFieldCount,
+                           long totalPartitionFieldCount, long bucketAllocationFailuresCount, MemoryStatus memoryStatus,
+                           Date timestamp, Date logTime) {
+        this.jobId = jobId;
+        this.modelBytes = modelBytes;
+        this.totalByFieldCount = totalByFieldCount;
+        this.totalOverFieldCount = totalOverFieldCount;
+        this.totalPartitionFieldCount = totalPartitionFieldCount;
+        this.bucketAllocationFailuresCount = bucketAllocationFailuresCount;
+        this.memoryStatus = memoryStatus;
+        this.timestamp = timestamp;
+        this.logTime = logTime;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder.field(Job.ID.getPreferredName(), jobId);
+        builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
+        builder.field(MODEL_BYTES_FIELD.getPreferredName(), modelBytes);
+        builder.field(TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName(), totalByFieldCount);
+        builder.field(TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName(), totalOverFieldCount);
+        builder.field(TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName(), totalPartitionFieldCount);
+        builder.field(BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName(), bucketAllocationFailuresCount);
+        builder.field(MEMORY_STATUS_FIELD.getPreferredName(), memoryStatus);
+        builder.timeField(LOG_TIME_FIELD.getPreferredName(), LOG_TIME_FIELD.getPreferredName() + "_string", logTime.getTime());
+        if (timestamp != null) {
+            builder.timeField(TIMESTAMP_FIELD.getPreferredName(), TIMESTAMP_FIELD.getPreferredName() + "_string", timestamp.getTime());
+        }
+
+        builder.endObject();
+        return builder;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public long getModelBytes() {
+        return modelBytes;
+    }
+
+    public long getTotalByFieldCount() {
+        return totalByFieldCount;
+    }
+
+    public long getTotalPartitionFieldCount() {
+        return totalPartitionFieldCount;
+    }
+
+    public long getTotalOverFieldCount() {
+        return totalOverFieldCount;
+    }
+
+    public long getBucketAllocationFailuresCount() {
+        return bucketAllocationFailuresCount;
+    }
+
+    public MemoryStatus getMemoryStatus() {
+        return memoryStatus;
+    }
+
+    /**
+     * The timestamp of the last processed record when this instance was created.
+     *
+     * @return The record time
+     */
+    public Date getTimestamp() {
+        return timestamp;
+    }
+
+    /**
+     * The wall clock time at the point when this instance was created.
+     *
+     * @return The wall clock time
+     */
+    public Date getLogTime() {
+        return logTime;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, modelBytes, totalByFieldCount, totalOverFieldCount, totalPartitionFieldCount,
+            this.bucketAllocationFailuresCount, memoryStatus, timestamp, logTime);
+    }
+
+    /**
+     * Compare all the fields.
+     */
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        ModelSizeStats that = (ModelSizeStats) other;
+
+        return this.modelBytes == that.modelBytes && this.totalByFieldCount == that.totalByFieldCount
+            && this.totalOverFieldCount == that.totalOverFieldCount && this.totalPartitionFieldCount == that.totalPartitionFieldCount
+            && this.bucketAllocationFailuresCount == that.bucketAllocationFailuresCount
+            && Objects.equals(this.memoryStatus, that.memoryStatus) && Objects.equals(this.timestamp, that.timestamp)
+            && Objects.equals(this.logTime, that.logTime)
+            && Objects.equals(this.jobId, that.jobId);
+    }
+
+    public static class Builder {
+
+        private final String jobId;
+        private long modelBytes;
+        private long totalByFieldCount;
+        private long totalOverFieldCount;
+        private long totalPartitionFieldCount;
+        private long bucketAllocationFailuresCount;
+        private MemoryStatus memoryStatus;
+        private Date timestamp;
+        private Date logTime;
+
+        public Builder(String jobId) {
+            this.jobId = jobId;
+            memoryStatus = MemoryStatus.OK;
+            logTime = new Date();
+        }
+
+        public Builder(ModelSizeStats modelSizeStats) {
+            this.jobId = modelSizeStats.jobId;
+            this.modelBytes = modelSizeStats.modelBytes;
+            this.totalByFieldCount = modelSizeStats.totalByFieldCount;
+            this.totalOverFieldCount = modelSizeStats.totalOverFieldCount;
+            this.totalPartitionFieldCount = modelSizeStats.totalPartitionFieldCount;
+            this.bucketAllocationFailuresCount = modelSizeStats.bucketAllocationFailuresCount;
+            this.memoryStatus = modelSizeStats.memoryStatus;
+            this.timestamp = modelSizeStats.timestamp;
+            this.logTime = modelSizeStats.logTime;
+        }
+
+        public Builder setModelBytes(long modelBytes) {
+            this.modelBytes = modelBytes;
+            return this;
+        }
+
+        public Builder setTotalByFieldCount(long totalByFieldCount) {
+            this.totalByFieldCount = totalByFieldCount;
+            return this;
+        }
+
+        public Builder setTotalPartitionFieldCount(long totalPartitionFieldCount) {
+            this.totalPartitionFieldCount = totalPartitionFieldCount;
+            return this;
+        }
+
+        public Builder setTotalOverFieldCount(long totalOverFieldCount) {
+            this.totalOverFieldCount = totalOverFieldCount;
+            return this;
+        }
+
+        public Builder setBucketAllocationFailuresCount(long bucketAllocationFailuresCount) {
+            this.bucketAllocationFailuresCount = bucketAllocationFailuresCount;
+            return this;
+        }
+
+        public Builder setMemoryStatus(MemoryStatus memoryStatus) {
+            Objects.requireNonNull(memoryStatus, "[" + MEMORY_STATUS_FIELD.getPreferredName() + "] must not be null");
+            this.memoryStatus = memoryStatus;
+            return this;
+        }
+
+        public Builder setTimestamp(Date timestamp) {
+            this.timestamp = timestamp;
+            return this;
+        }
+
+        public Builder setLogTime(Date logTime) {
+            this.logTime = logTime;
+            return this;
+        }
+
+        public ModelSizeStats build() {
+            return new ModelSizeStats(jobId, modelBytes, totalByFieldCount, totalOverFieldCount, totalPartitionFieldCount,
+                bucketAllocationFailuresCount, memoryStatus, timestamp, logTime);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshot.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshot.java
new file mode 100644
index 0000000000000..2b9957f9bc756
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshot.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.process;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.Objects;
+
+/**
+ * ModelSnapshot Result POJO
+ */
+public class ModelSnapshot implements ToXContentObject {
+    /**
+     * Field Names
+     */
+    public static final ParseField TIMESTAMP = new ParseField("timestamp");
+    public static final ParseField DESCRIPTION = new ParseField("description");
+    public static final ParseField SNAPSHOT_DOC_COUNT = new ParseField("snapshot_doc_count");
+    public static final ParseField LATEST_RECORD_TIME = new ParseField("latest_record_time_stamp");
+    public static final ParseField LATEST_RESULT_TIME = new ParseField("latest_result_time_stamp");
+    public static final ParseField QUANTILES = new ParseField("quantiles");
+    public static final ParseField RETAIN = new ParseField("retain");
+    public static final ParseField MIN_VERSION = new ParseField("min_version");
+    public static final ParseField SNAPSHOT_ID = new ParseField("snapshot_id");
+
+    public static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>("model_snapshot", true, Builder::new);
+
+    static {
+        PARSER.declareString(Builder::setJobId, Job.ID);
+        PARSER.declareString(Builder::setMinVersion, MIN_VERSION);
+        PARSER.declareField(Builder::setTimestamp,
+            (p) -> TimeUtil.parseTimeField(p, TIMESTAMP.getPreferredName()),
+            TIMESTAMP,
+            ValueType.VALUE);
+        PARSER.declareString(Builder::setDescription, DESCRIPTION);
+        PARSER.declareString(Builder::setSnapshotId, SNAPSHOT_ID);
+        PARSER.declareInt(Builder::setSnapshotDocCount, SNAPSHOT_DOC_COUNT);
+        PARSER.declareObject(Builder::setModelSizeStats, ModelSizeStats.PARSER,
+            ModelSizeStats.RESULT_TYPE_FIELD);
+        PARSER.declareField(Builder::setLatestRecordTimeStamp,
+            (p) -> TimeUtil.parseTimeField(p, LATEST_RECORD_TIME.getPreferredName()),
+            LATEST_RECORD_TIME,
+            ValueType.VALUE);
+        PARSER.declareField(Builder::setLatestResultTimeStamp,
+            (p) -> TimeUtil.parseTimeField(p, LATEST_RESULT_TIME.getPreferredName()),
+            LATEST_RESULT_TIME,
+            ValueType.VALUE);
+        PARSER.declareObject(Builder::setQuantiles, Quantiles.PARSER, QUANTILES);
+        PARSER.declareBoolean(Builder::setRetain, RETAIN);
+    }
+
+
+    private final String jobId;
+
+    /**
+     * The minimum version a node should have to be able
+     * to read this model snapshot.
+     */
+    private final Version minVersion;
+
+    private final Date timestamp;
+    private final String description;
+    private final String snapshotId;
+    private final int snapshotDocCount;
+    private final ModelSizeStats modelSizeStats;
+    private final Date latestRecordTimeStamp;
+    private final Date latestResultTimeStamp;
+    private final Quantiles quantiles;
+    private final boolean retain;
+
+
+    private ModelSnapshot(String jobId, Version minVersion, Date timestamp, String description, String snapshotId, int snapshotDocCount,
+                          ModelSizeStats modelSizeStats, Date latestRecordTimeStamp, Date latestResultTimeStamp, Quantiles quantiles,
+                          boolean retain) {
+        this.jobId = jobId;
+        this.minVersion = minVersion;
+        this.timestamp = timestamp;
+        this.description = description;
+        this.snapshotId = snapshotId;
+        this.snapshotDocCount = snapshotDocCount;
+        this.modelSizeStats = modelSizeStats;
+        this.latestRecordTimeStamp = latestRecordTimeStamp;
+        this.latestResultTimeStamp = latestResultTimeStamp;
+        this.quantiles = quantiles;
+        this.retain = retain;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Job.ID.getPreferredName(), jobId);
+        builder.field(MIN_VERSION.getPreferredName(), minVersion);
+        if (timestamp != null) {
+            builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        }
+        if (description != null) {
+            builder.field(DESCRIPTION.getPreferredName(), description);
+        }
+        if (snapshotId != null) {
+            builder.field(SNAPSHOT_ID.getPreferredName(), snapshotId);
+        }
+        builder.field(SNAPSHOT_DOC_COUNT.getPreferredName(), snapshotDocCount);
+        if (modelSizeStats != null) {
+            builder.field(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName(), modelSizeStats);
+        }
+        if (latestRecordTimeStamp != null) {
+            builder.timeField(LATEST_RECORD_TIME.getPreferredName(), LATEST_RECORD_TIME.getPreferredName() + "_string",
+                latestRecordTimeStamp.getTime());
+        }
+        if (latestResultTimeStamp != null) {
+            builder.timeField(LATEST_RESULT_TIME.getPreferredName(), LATEST_RESULT_TIME.getPreferredName() + "_string",
+                latestResultTimeStamp.getTime());
+        }
+        if (quantiles != null) {
+            builder.field(QUANTILES.getPreferredName(), quantiles);
+        }
+        builder.field(RETAIN.getPreferredName(), retain);
+        builder.endObject();
+        return builder;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public Version getMinVersion() {
+        return minVersion;
+    }
+
+    public Date getTimestamp() {
+        return timestamp;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public String getSnapshotId() {
+        return snapshotId;
+    }
+
+    public int getSnapshotDocCount() {
+        return snapshotDocCount;
+    }
+
+    public ModelSizeStats getModelSizeStats() {
+        return modelSizeStats;
+    }
+
+    public Quantiles getQuantiles() {
+        return quantiles;
+    }
+
+    public Date getLatestRecordTimeStamp() {
+        return latestRecordTimeStamp;
+    }
+
+    public Date getLatestResultTimeStamp() {
+        return latestResultTimeStamp;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, minVersion, timestamp, description, snapshotId, quantiles, snapshotDocCount, modelSizeStats,
+            latestRecordTimeStamp, latestResultTimeStamp, retain);
+    }
+
+    /**
+     * Compare all the fields.
+     */
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        ModelSnapshot that = (ModelSnapshot) other;
+
+        return Objects.equals(this.jobId, that.jobId)
+            && Objects.equals(this.minVersion, that.minVersion)
+            && Objects.equals(this.timestamp, that.timestamp)
+            && Objects.equals(this.description, that.description)
+            && Objects.equals(this.snapshotId, that.snapshotId)
+            && this.snapshotDocCount == that.snapshotDocCount
+            && Objects.equals(this.modelSizeStats, that.modelSizeStats)
+            && Objects.equals(this.quantiles, that.quantiles)
+            && Objects.equals(this.latestRecordTimeStamp, that.latestRecordTimeStamp)
+            && Objects.equals(this.latestResultTimeStamp, that.latestResultTimeStamp)
+            && this.retain == that.retain;
+    }
+
+    public static class Builder {
+        private String jobId;
+
+        // Stored snapshot documents created prior to 6.3.0 will have no
+        // value for min_version. We default it to 5.5.0 as there were
+        // no model changes between 5.5.0 and 6.3.0.
+        private Version minVersion = Version.V_5_5_0;
+
+        private Date timestamp;
+        private String description;
+        private String snapshotId;
+        private int snapshotDocCount;
+        private ModelSizeStats modelSizeStats;
+        private Date latestRecordTimeStamp;
+        private Date latestResultTimeStamp;
+        private Quantiles quantiles;
+        private boolean retain;
+
+
+        public Builder() {
+        }
+
+        public Builder(String jobId) {
+            this.jobId = jobId;
+        }
+
+        public Builder(ModelSnapshot modelSnapshot) {
+            this.jobId = modelSnapshot.jobId;
+            this.timestamp = modelSnapshot.timestamp;
+            this.description = modelSnapshot.description;
+            this.snapshotId = modelSnapshot.snapshotId;
+            this.snapshotDocCount = modelSnapshot.snapshotDocCount;
+            this.modelSizeStats = modelSnapshot.modelSizeStats;
+            this.latestRecordTimeStamp = modelSnapshot.latestRecordTimeStamp;
+            this.latestResultTimeStamp = modelSnapshot.latestResultTimeStamp;
+            this.quantiles = modelSnapshot.quantiles;
+            this.retain = modelSnapshot.retain;
+            this.minVersion = modelSnapshot.minVersion;
+        }
+
+        public Builder setJobId(String jobId) {
+            this.jobId = jobId;
+            return this;
+        }
+
+        Builder setMinVersion(Version minVersion) {
+            this.minVersion = minVersion;
+            return this;
+        }
+
+        Builder setMinVersion(String minVersion) {
+            this.minVersion = Version.fromString(minVersion);
+            return this;
+        }
+
+        public Builder setTimestamp(Date timestamp) {
+            this.timestamp = timestamp;
+            return this;
+        }
+
+        public Builder setDescription(String description) {
+            this.description = description;
+            return this;
+        }
+
+        public Builder setSnapshotId(String snapshotId) {
+            this.snapshotId = snapshotId;
+            return this;
+        }
+
+        public Builder setSnapshotDocCount(int snapshotDocCount) {
+            this.snapshotDocCount = snapshotDocCount;
+            return this;
+        }
+
+        public Builder setModelSizeStats(ModelSizeStats.Builder modelSizeStats) {
+            this.modelSizeStats = modelSizeStats.build();
+            return this;
+        }
+
+        public Builder setModelSizeStats(ModelSizeStats modelSizeStats) {
+            this.modelSizeStats = modelSizeStats;
+            return this;
+        }
+
+        public Builder setLatestRecordTimeStamp(Date latestRecordTimeStamp) {
+            this.latestRecordTimeStamp = latestRecordTimeStamp;
+            return this;
+        }
+
+        public Builder setLatestResultTimeStamp(Date latestResultTimeStamp) {
+            this.latestResultTimeStamp = latestResultTimeStamp;
+            return this;
+        }
+
+        public Builder setQuantiles(Quantiles quantiles) {
+            this.quantiles = quantiles;
+            return this;
+        }
+
+        public Builder setRetain(boolean value) {
+            this.retain = value;
+            return this;
+        }
+
+        public ModelSnapshot build() {
+            return new ModelSnapshot(jobId, minVersion, timestamp, description, snapshotId, snapshotDocCount, modelSizeStats,
+                latestRecordTimeStamp, latestResultTimeStamp, quantiles, retain);
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/Quantiles.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/Quantiles.java
new file mode 100644
index 0000000000000..1c047d6c30284
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/Quantiles.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.process;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.Objects;
+
+/**
+ * Quantiles Result POJO
+ */
+public class Quantiles implements ToXContentObject {
+
+    /**
+     * Field Names
+     */
+    public static final ParseField TIMESTAMP = new ParseField("timestamp");
+    public static final ParseField QUANTILE_STATE = new ParseField("quantile_state");
+
+    public static final ConstructingObjectParser<Quantiles, Void> PARSER =
+        new ConstructingObjectParser<>("quantiles", true, a -> new Quantiles((String) a[0], (Date) a[1], (String) a[2]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> new Date(p.longValue()), TIMESTAMP, ValueType.LONG);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), QUANTILE_STATE);
+    }
+
+    private final String jobId;
+    private final Date timestamp;
+    private final String quantileState;
+
+    public Quantiles(String jobId, Date timestamp, String quantileState) {
+        this.jobId = jobId;
+        this.timestamp = Objects.requireNonNull(timestamp);
+        this.quantileState = Objects.requireNonNull(quantileState);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Job.ID.getPreferredName(), jobId);
+        if (timestamp != null) {
+            builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime());
+        }
+        if (quantileState != null) {
+            builder.field(QUANTILE_STATE.getPreferredName(), quantileState);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public Date getTimestamp() {
+        return timestamp;
+    }
+
+    public String getQuantileState() {
+        return quantileState;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, timestamp, quantileState);
+    }
+
+    /**
+     * Compare all the fields.
+     */
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        Quantiles that = (Quantiles) other;
+
+        return Objects.equals(this.jobId, that.jobId) && Objects.equals(this.timestamp, that.timestamp)
+            && Objects.equals(this.quantileState, that.quantileState);
+    }
+}
+
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyCause.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyCause.java
new file mode 100644
index 0000000000000..7ad57b24fcbdc
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyCause.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Anomaly Cause POJO.
+ * Used as a nested level inside population anomaly records.
+ */
+public class AnomalyCause implements ToXContentObject {
+
+    public static final ParseField ANOMALY_CAUSE = new ParseField("anomaly_cause");
+
+    /**
+     * Result fields
+     */
+    public static final ParseField PROBABILITY = new ParseField("probability");
+    public static final ParseField OVER_FIELD_NAME = new ParseField("over_field_name");
+    public static final ParseField OVER_FIELD_VALUE = new ParseField("over_field_value");
+    public static final ParseField BY_FIELD_NAME = new ParseField("by_field_name");
+    public static final ParseField BY_FIELD_VALUE = new ParseField("by_field_value");
+    public static final ParseField CORRELATED_BY_FIELD_VALUE = new ParseField("correlated_by_field_value");
+    public static final ParseField PARTITION_FIELD_NAME = new ParseField("partition_field_name");
+    public static final ParseField PARTITION_FIELD_VALUE = new ParseField("partition_field_value");
+    public static final ParseField FUNCTION = new ParseField("function");
+    public static final ParseField FUNCTION_DESCRIPTION = new ParseField("function_description");
+    public static final ParseField TYPICAL = new ParseField("typical");
+    public static final ParseField ACTUAL = new ParseField("actual");
+    public static final ParseField INFLUENCERS = new ParseField("influencers");
+
+    /**
+     * Metric Results
+     */
+    public static final ParseField FIELD_NAME = new ParseField("field_name");
+
+    public static final ObjectParser<AnomalyCause, Void> PARSER =
+        new ObjectParser<>(ANOMALY_CAUSE.getPreferredName(), true, AnomalyCause::new);
+
+    static {
+        PARSER.declareDouble(AnomalyCause::setProbability, PROBABILITY);
+        PARSER.declareString(AnomalyCause::setByFieldName, BY_FIELD_NAME);
+        PARSER.declareString(AnomalyCause::setByFieldValue, BY_FIELD_VALUE);
+        PARSER.declareString(AnomalyCause::setCorrelatedByFieldValue, CORRELATED_BY_FIELD_VALUE);
+        PARSER.declareString(AnomalyCause::setPartitionFieldName, PARTITION_FIELD_NAME);
+        PARSER.declareString(AnomalyCause::setPartitionFieldValue, PARTITION_FIELD_VALUE);
+        PARSER.declareString(AnomalyCause::setFunction, FUNCTION);
+        PARSER.declareString(AnomalyCause::setFunctionDescription, FUNCTION_DESCRIPTION);
+        PARSER.declareDoubleArray(AnomalyCause::setTypical, TYPICAL);
+        PARSER.declareDoubleArray(AnomalyCause::setActual, ACTUAL);
+        PARSER.declareString(AnomalyCause::setFieldName, FIELD_NAME);
+        PARSER.declareString(AnomalyCause::setOverFieldName, OVER_FIELD_NAME);
+        PARSER.declareString(AnomalyCause::setOverFieldValue, OVER_FIELD_VALUE);
+        PARSER.declareObjectArray(AnomalyCause::setInfluencers, Influence.PARSER, INFLUENCERS);
+    }
+
+    private double probability;
+    private String byFieldName;
+    private String byFieldValue;
+    private String correlatedByFieldValue;
+    private String partitionFieldName;
+    private String partitionFieldValue;
+    private String function;
+    private String functionDescription;
+    private List<Double> typical;
+    private List<Double> actual;
+    private String fieldName;
+    private String overFieldName;
+    private String overFieldValue;
+
+    private List<Influence> influencers;
+
+    AnomalyCause() {
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(PROBABILITY.getPreferredName(), probability);
+        if (byFieldName != null) {
+            builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName);
+        }
+        if (byFieldValue != null) {
+            builder.field(BY_FIELD_VALUE.getPreferredName(), byFieldValue);
+        }
+        if (correlatedByFieldValue != null) {
+            builder.field(CORRELATED_BY_FIELD_VALUE.getPreferredName(), correlatedByFieldValue);
+        }
+        if (partitionFieldName != null) {
+            builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName);
+        }
+        if (partitionFieldValue != null) {
+            builder.field(PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue);
+        }
+        if (function != null) {
+            builder.field(FUNCTION.getPreferredName(), function);
+        }
+        if (functionDescription != null) {
+            builder.field(FUNCTION_DESCRIPTION.getPreferredName(), functionDescription);
+        }
+        if (typical != null) {
+            builder.field(TYPICAL.getPreferredName(), typical);
+        }
+        if (actual != null) {
+            builder.field(ACTUAL.getPreferredName(), actual);
+        }
+        if (fieldName != null) {
+            builder.field(FIELD_NAME.getPreferredName(), fieldName);
+        }
+        if (overFieldName != null) {
+            builder.field(OVER_FIELD_NAME.getPreferredName(), overFieldName);
+        }
+        if (overFieldValue != null) {
+            builder.field(OVER_FIELD_VALUE.getPreferredName(), overFieldValue);
+        }
+        if (influencers != null) {
+            builder.field(INFLUENCERS.getPreferredName(), influencers);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public double getProbability() {
+        return probability;
+    }
+
+    void setProbability(double value) {
+        probability = value;
+    }
+
+    public String getByFieldName() {
+        return byFieldName;
+    }
+
+    void setByFieldName(String value) {
+        byFieldName = value;
+    }
+
+    public String getByFieldValue() {
+        return byFieldValue;
+    }
+
+    void setByFieldValue(String value) {
+        byFieldValue = value;
+    }
+
+    public String getCorrelatedByFieldValue() {
+        return correlatedByFieldValue;
+    }
+
+    void setCorrelatedByFieldValue(String value) {
+        correlatedByFieldValue = value;
+    }
+
+    public String getPartitionFieldName() {
+        return partitionFieldName;
+    }
+
+    void setPartitionFieldName(String field) {
+        partitionFieldName = field;
+    }
+
+    public String getPartitionFieldValue() {
+        return partitionFieldValue;
+    }
+
+    void setPartitionFieldValue(String value) {
+        partitionFieldValue = value;
+    }
+
+    public String getFunction() {
+        return function;
+    }
+
+    void setFunction(String name) {
+        function = name;
+    }
+
+    public String getFunctionDescription() {
+        return functionDescription;
+    }
+
+    void setFunctionDescription(String functionDescription) {
+        this.functionDescription = functionDescription;
+    }
+
+    public List<Double> getTypical() {
+        return typical;
+    }
+
+    void setTypical(List<Double> typical) {
+        this.typical = Collections.unmodifiableList(typical);
+    }
+
+    public List<Double> getActual() {
+        return actual;
+    }
+
+    void setActual(List<Double> actual) {
+        this.actual = Collections.unmodifiableList(actual);
+    }
+
+    public String getFieldName() {
+        return fieldName;
+    }
+
+    void setFieldName(String field) {
+        fieldName = field;
+    }
+
+    public String getOverFieldName() {
+        return overFieldName;
+    }
+
+    void setOverFieldName(String name) {
+        overFieldName = name;
+    }
+
+    public String getOverFieldValue() {
+        return overFieldValue;
+    }
+
+    void setOverFieldValue(String value) {
+        overFieldValue = value;
+    }
+
+    public List<Influence> getInfluencers() {
+        return influencers;
+    }
+
+    void setInfluencers(List<Influence> influencers) {
+        this.influencers = Collections.unmodifiableList(influencers);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(probability, actual, typical, byFieldName, byFieldValue, correlatedByFieldValue, fieldName, function,
+            functionDescription, overFieldName, overFieldValue, partitionFieldName, partitionFieldValue, influencers);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        AnomalyCause that = (AnomalyCause)other;
+
+        return this.probability == that.probability &&
+            Objects.equals(this.typical, that.typical) &&
+            Objects.equals(this.actual, that.actual) &&
+            Objects.equals(this.function, that.function) &&
+            Objects.equals(this.functionDescription, that.functionDescription) &&
+            Objects.equals(this.fieldName, that.fieldName) &&
+            Objects.equals(this.byFieldName, that.byFieldName) &&
+            Objects.equals(this.byFieldValue, that.byFieldValue) &&
+            Objects.equals(this.correlatedByFieldValue, that.correlatedByFieldValue) &&
+            Objects.equals(this.partitionFieldName, that.partitionFieldName) &&
+            Objects.equals(this.partitionFieldValue, that.partitionFieldValue) &&
+            Objects.equals(this.overFieldName, that.overFieldName) &&
+            Objects.equals(this.overFieldValue, that.overFieldValue) &&
+            Objects.equals(this.influencers, that.influencers);
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyRecord.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyRecord.java
new file mode 100644
index 0000000000000..4747f3a48bdc8
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyRecord.java
@@ -0,0 +1,429 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.io.IOException;
+import java.time.format.DateTimeFormatter;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Anomaly Record POJO.
+ * Uses the object wrappers Boolean and Double so <code>null</code> values
+ * can be returned if the members have not been set.
+ */
+public class AnomalyRecord implements ToXContentObject {
+
+    /**
+     * Result type
+     */
+    public static final String RESULT_TYPE_VALUE = "record";
+
+    /**
+     * Result fields (all detector types)
+     */
+    public static final ParseField PROBABILITY = new ParseField("probability");
+    public static final ParseField DETECTOR_INDEX = new ParseField("detector_index");
+    public static final ParseField BY_FIELD_NAME = new ParseField("by_field_name");
+    public static final ParseField BY_FIELD_VALUE = new ParseField("by_field_value");
+    public static final ParseField CORRELATED_BY_FIELD_VALUE = new ParseField("correlated_by_field_value");
+    public static final ParseField PARTITION_FIELD_NAME = new ParseField("partition_field_name");
+    public static final ParseField PARTITION_FIELD_VALUE = new ParseField("partition_field_value");
+    public static final ParseField FUNCTION = new ParseField("function");
+    public static final ParseField FUNCTION_DESCRIPTION = new ParseField("function_description");
+    public static final ParseField TYPICAL = new ParseField("typical");
+    public static final ParseField ACTUAL = new ParseField("actual");
+    public static final ParseField INFLUENCERS = new ParseField("influencers");
+    public static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
+
+    // Used for QueryPage
+    public static final ParseField RESULTS_FIELD = new ParseField("records");
+
+    /**
+     * Metric Results (including population metrics)
+     */
+    public static final ParseField FIELD_NAME = new ParseField("field_name");
+
+    /**
+     * Population results
+     */
+    public static final ParseField OVER_FIELD_NAME = new ParseField("over_field_name");
+    public static final ParseField OVER_FIELD_VALUE = new ParseField("over_field_value");
+    public static final ParseField CAUSES = new ParseField("causes");
+
+    /**
+     * Normalization
+     */
+    public static final ParseField RECORD_SCORE = new ParseField("record_score");
+    public static final ParseField INITIAL_RECORD_SCORE = new ParseField("initial_record_score");
+
+    public static final ConstructingObjectParser<AnomalyRecord, Void> PARSER =
+        new ConstructingObjectParser<>(RESULT_TYPE_VALUE, true, a -> new AnomalyRecord((String) a[0], (Date) a[1], (long) a[2]));
+
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
+        PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> {
+            if (p.currentToken() == Token.VALUE_NUMBER) {
+                return new Date(p.longValue());
+            } else if (p.currentToken() == Token.VALUE_STRING) {
+                return new Date(DateFormatters.toZonedDateTime(DateTimeFormatter.ISO_INSTANT.parse(p.text())).toInstant().toEpochMilli());
+            }
+            throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for ["
+                    + Result.TIMESTAMP.getPreferredName() + "]");
+        }, Result.TIMESTAMP, ValueType.VALUE);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN);
+        PARSER.declareString((anomalyRecord, s) -> {}, Result.RESULT_TYPE);
+        PARSER.declareDouble(AnomalyRecord::setProbability, PROBABILITY);
+        PARSER.declareDouble(AnomalyRecord::setRecordScore, RECORD_SCORE);
+        PARSER.declareDouble(AnomalyRecord::setInitialRecordScore, INITIAL_RECORD_SCORE);
+        PARSER.declareInt(AnomalyRecord::setDetectorIndex, DETECTOR_INDEX);
+        PARSER.declareBoolean(AnomalyRecord::setInterim, Result.IS_INTERIM);
+        PARSER.declareString(AnomalyRecord::setByFieldName, BY_FIELD_NAME);
+        PARSER.declareString(AnomalyRecord::setByFieldValue, BY_FIELD_VALUE);
+        PARSER.declareString(AnomalyRecord::setCorrelatedByFieldValue, CORRELATED_BY_FIELD_VALUE);
+        PARSER.declareString(AnomalyRecord::setPartitionFieldName, PARTITION_FIELD_NAME);
+        PARSER.declareString(AnomalyRecord::setPartitionFieldValue, PARTITION_FIELD_VALUE);
+        PARSER.declareString(AnomalyRecord::setFunction, FUNCTION);
+        PARSER.declareString(AnomalyRecord::setFunctionDescription, FUNCTION_DESCRIPTION);
+        PARSER.declareDoubleArray(AnomalyRecord::setTypical, TYPICAL);
+        PARSER.declareDoubleArray(AnomalyRecord::setActual, ACTUAL);
+        PARSER.declareString(AnomalyRecord::setFieldName, FIELD_NAME);
+        PARSER.declareString(AnomalyRecord::setOverFieldName, OVER_FIELD_NAME);
+        PARSER.declareString(AnomalyRecord::setOverFieldValue, OVER_FIELD_VALUE);
+        PARSER.declareObjectArray(AnomalyRecord::setCauses, AnomalyCause.PARSER, CAUSES);
+        PARSER.declareObjectArray(AnomalyRecord::setInfluencers, Influence.PARSER, INFLUENCERS);
+    }
+
+    private final String jobId;
+    private int detectorIndex;
+    private double probability;
+    private String byFieldName;
+    private String byFieldValue;
+    private String correlatedByFieldValue;
+    private String partitionFieldName;
+    private String partitionFieldValue;
+    private String function;
+    private String functionDescription;
+    private List<Double> typical;
+    private List<Double> actual;
+    private boolean isInterim;
+
+    private String fieldName;
+
+    private String overFieldName;
+    private String overFieldValue;
+    private List<AnomalyCause> causes;
+
+    private double recordScore;
+
+    private double initialRecordScore;
+
+    private final Date timestamp;
+    private final long bucketSpan;
+
+    private List<Influence> influences;
+
+    AnomalyRecord(String jobId, Date timestamp, long bucketSpan) {
+        this.jobId = jobId;
+        this.timestamp = Objects.requireNonNull(timestamp);
+        this.bucketSpan = bucketSpan;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Job.ID.getPreferredName(), jobId);
+        builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
+        builder.field(PROBABILITY.getPreferredName(), probability);
+        builder.field(RECORD_SCORE.getPreferredName(), recordScore);
+        builder.field(INITIAL_RECORD_SCORE.getPreferredName(), initialRecordScore);
+        builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
+        builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex);
+        builder.field(Result.IS_INTERIM.getPreferredName(), isInterim);
+        builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        if (byFieldName != null) {
+            builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName);
+        }
+        if (byFieldValue != null) {
+            builder.field(BY_FIELD_VALUE.getPreferredName(), byFieldValue);
+        }
+        if (correlatedByFieldValue != null) {
+            builder.field(CORRELATED_BY_FIELD_VALUE.getPreferredName(), correlatedByFieldValue);
+        }
+        if (partitionFieldName != null) {
+            builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName);
+        }
+        if (partitionFieldValue != null) {
+            builder.field(PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue);
+        }
+        if (function != null) {
+            builder.field(FUNCTION.getPreferredName(), function);
+        }
+        if (functionDescription != null) {
+            builder.field(FUNCTION_DESCRIPTION.getPreferredName(), functionDescription);
+        }
+        if (typical != null) {
+            builder.field(TYPICAL.getPreferredName(), typical);
+        }
+        if (actual != null) {
+            builder.field(ACTUAL.getPreferredName(), actual);
+        }
+        if (fieldName != null) {
+            builder.field(FIELD_NAME.getPreferredName(), fieldName);
+        }
+        if (overFieldName != null) {
+            builder.field(OVER_FIELD_NAME.getPreferredName(), overFieldName);
+        }
+        if (overFieldValue != null) {
+            builder.field(OVER_FIELD_VALUE.getPreferredName(), overFieldValue);
+        }
+        if (causes != null) {
+            builder.field(CAUSES.getPreferredName(), causes);
+        }
+        if (influences != null) {
+            builder.field(INFLUENCERS.getPreferredName(), influences);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public String getJobId() {
+        return this.jobId;
+    }
+
+    public int getDetectorIndex() {
+        return detectorIndex;
+    }
+
+    void setDetectorIndex(int detectorIndex) {
+        this.detectorIndex = detectorIndex;
+    }
+
+    public double getRecordScore() {
+        return recordScore;
+    }
+
+    void setRecordScore(double recordScore) {
+        this.recordScore = recordScore;
+    }
+
+    public double getInitialRecordScore() {
+        return initialRecordScore;
+    }
+
+    void setInitialRecordScore(double initialRecordScore) {
+        this.initialRecordScore = initialRecordScore;
+    }
+
+    public Date getTimestamp() {
+        return timestamp;
+    }
+
+    /**
+     * Bucketspan expressed in seconds
+     */
+    public long getBucketSpan() {
+        return bucketSpan;
+    }
+
+    public double getProbability() {
+        return probability;
+    }
+
+    void setProbability(double value) {
+        probability = value;
+    }
+
+    public String getByFieldName() {
+        return byFieldName;
+    }
+
+    void setByFieldName(String value) {
+        byFieldName = value;
+    }
+
+    public String getByFieldValue() {
+        return byFieldValue;
+    }
+
+    void setByFieldValue(String value) {
+        byFieldValue = value;
+    }
+
+    public String getCorrelatedByFieldValue() {
+        return correlatedByFieldValue;
+    }
+
+    void setCorrelatedByFieldValue(String value) {
+        correlatedByFieldValue = value;
+    }
+
+    public String getPartitionFieldName() {
+        return partitionFieldName;
+    }
+
+    void setPartitionFieldName(String field) {
+        partitionFieldName = field;
+    }
+
+    public String getPartitionFieldValue() {
+        return partitionFieldValue;
+    }
+
+    void setPartitionFieldValue(String value) {
+        partitionFieldValue = value;
+    }
+
+    public String getFunction() {
+        return function;
+    }
+
+    void setFunction(String name) {
+        function = name;
+    }
+
+    public String getFunctionDescription() {
+        return functionDescription;
+    }
+
+    void setFunctionDescription(String functionDescription) {
+        this.functionDescription = functionDescription;
+    }
+
+    public List<Double> getTypical() {
+        return typical;
+    }
+
+    void setTypical(List<Double> typical) {
+        this.typical = Collections.unmodifiableList(typical);
+    }
+
+    public List<Double> getActual() {
+        return actual;
+    }
+
+    void setActual(List<Double> actual) {
+        this.actual = Collections.unmodifiableList(actual);
+    }
+
+    public boolean isInterim() {
+        return isInterim;
+    }
+
+    void setInterim(boolean isInterim) {
+        this.isInterim = isInterim;
+    }
+
+    public String getFieldName() {
+        return fieldName;
+    }
+
+    void setFieldName(String field) {
+        fieldName = field;
+    }
+
+    public String getOverFieldName() {
+        return overFieldName;
+    }
+
+    void setOverFieldName(String name) {
+        overFieldName = name;
+    }
+
+    public String getOverFieldValue() {
+        return overFieldValue;
+    }
+
+    void setOverFieldValue(String value) {
+        overFieldValue = value;
+    }
+
+    public List<AnomalyCause> getCauses() {
+        return causes;
+    }
+
+    void setCauses(List<AnomalyCause> causes) {
+        this.causes = Collections.unmodifiableList(causes);
+    }
+
+    public List<Influence> getInfluencers() {
+        return influences;
+    }
+
+    void setInfluencers(List<Influence> influencers) {
+        this.influences = Collections.unmodifiableList(influencers);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, detectorIndex, bucketSpan, probability, recordScore,
+                initialRecordScore, typical, actual,function, functionDescription, fieldName,
+                byFieldName, byFieldValue, correlatedByFieldValue, partitionFieldName,
+                partitionFieldValue, overFieldName, overFieldValue, timestamp, isInterim,
+                causes, influences, jobId);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        AnomalyRecord that = (AnomalyRecord) other;
+
+        return Objects.equals(this.jobId, that.jobId)
+            && this.detectorIndex == that.detectorIndex
+            && this.bucketSpan == that.bucketSpan
+            && this.probability == that.probability
+            && this.recordScore == that.recordScore
+            && this.initialRecordScore == that.initialRecordScore
+            && Objects.deepEquals(this.typical, that.typical)
+            && Objects.deepEquals(this.actual, that.actual)
+            && Objects.equals(this.function, that.function)
+            && Objects.equals(this.functionDescription, that.functionDescription)
+            && Objects.equals(this.fieldName, that.fieldName)
+            && Objects.equals(this.byFieldName, that.byFieldName)
+            && Objects.equals(this.byFieldValue, that.byFieldValue)
+            && Objects.equals(this.correlatedByFieldValue, that.correlatedByFieldValue)
+            && Objects.equals(this.partitionFieldName, that.partitionFieldName)
+            && Objects.equals(this.partitionFieldValue, that.partitionFieldValue)
+            && Objects.equals(this.overFieldName, that.overFieldName)
+            && Objects.equals(this.overFieldValue, that.overFieldValue)
+            && Objects.equals(this.timestamp, that.timestamp)
+            && Objects.equals(this.isInterim, that.isInterim)
+            && Objects.equals(this.causes, that.causes)
+            && Objects.equals(this.influences, that.influences);
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Bucket.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Bucket.java
new file mode 100644
index 0000000000000..cbaf83abbad40
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Bucket.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.io.IOException;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Bucket Result POJO
+ */
+public class Bucket implements ToXContentObject {
+
+    public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score");
+    public static final ParseField INITIAL_ANOMALY_SCORE = new ParseField("initial_anomaly_score");
+    public static final ParseField EVENT_COUNT = new ParseField("event_count");
+    public static final ParseField RECORDS = new ParseField("records");
+    public static final ParseField BUCKET_INFLUENCERS = new ParseField("bucket_influencers");
+    public static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
+    public static final ParseField PROCESSING_TIME_MS = new ParseField("processing_time_ms");
+    public static final ParseField SCHEDULED_EVENTS = new ParseField("scheduled_events");
+
+    // Used for QueryPage
+    public static final ParseField RESULTS_FIELD = new ParseField("buckets");
+
+    /**
+     * Result type
+     */
+    public static final String RESULT_TYPE_VALUE = "bucket";
+    public static final ParseField RESULT_TYPE_FIELD = new ParseField(RESULT_TYPE_VALUE);
+
+    public static final ConstructingObjectParser<Bucket, Void> PARSER =
+        new ConstructingObjectParser<>(RESULT_TYPE_VALUE, true, a -> new Bucket((String) a[0], (Date) a[1], (long) a[2]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
+        PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> {
+            if (p.currentToken() == Token.VALUE_NUMBER) {
+                return new Date(p.longValue());
+            } else if (p.currentToken() == Token.VALUE_STRING) {
+                return new Date(DateFormatters.toZonedDateTime(DateTimeFormatter.ISO_INSTANT.parse(p.text())).toInstant().toEpochMilli());
+            }
+            throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for ["
+                    + Result.TIMESTAMP.getPreferredName() + "]");
+        }, Result.TIMESTAMP, ValueType.VALUE);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN);
+        PARSER.declareDouble(Bucket::setAnomalyScore, ANOMALY_SCORE);
+        PARSER.declareDouble(Bucket::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE);
+        PARSER.declareBoolean(Bucket::setInterim, Result.IS_INTERIM);
+        PARSER.declareLong(Bucket::setEventCount, EVENT_COUNT);
+        PARSER.declareObjectArray(Bucket::setRecords, AnomalyRecord.PARSER, RECORDS);
+        PARSER.declareObjectArray(Bucket::setBucketInfluencers, BucketInfluencer.PARSER, BUCKET_INFLUENCERS);
+        PARSER.declareLong(Bucket::setProcessingTimeMs, PROCESSING_TIME_MS);
+        PARSER.declareString((bucket, s) -> {}, Result.RESULT_TYPE);
+        PARSER.declareStringArray(Bucket::setScheduledEvents, SCHEDULED_EVENTS);
+    }
+
+    private final String jobId;
+    private final Date timestamp;
+    private final long bucketSpan;
+    private double anomalyScore;
+    private double initialAnomalyScore;
+    private List<AnomalyRecord> records = new ArrayList<>();
+    private long eventCount;
+    private boolean isInterim;
+    private List<BucketInfluencer> bucketInfluencers = new ArrayList<>(); // Can't use emptyList as might be appended to
+    private long processingTimeMs;
+    private List<String> scheduledEvents = Collections.emptyList();
+
+    Bucket(String jobId, Date timestamp, long bucketSpan) {
+        this.jobId = jobId;
+        this.timestamp = Objects.requireNonNull(timestamp);
+        this.bucketSpan = bucketSpan;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Job.ID.getPreferredName(), jobId);
+        builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore);
+        builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
+        builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore);
+        if (records.isEmpty() == false) {
+            builder.field(RECORDS.getPreferredName(), records);
+        }
+        builder.field(EVENT_COUNT.getPreferredName(), eventCount);
+        builder.field(Result.IS_INTERIM.getPreferredName(), isInterim);
+        builder.field(BUCKET_INFLUENCERS.getPreferredName(), bucketInfluencers);
+        builder.field(PROCESSING_TIME_MS.getPreferredName(), processingTimeMs);
+        if (scheduledEvents.isEmpty() == false) {
+            builder.field(SCHEDULED_EVENTS.getPreferredName(), scheduledEvents);
+        }
+        builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
+        builder.endObject();
+        return builder;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public Date getTimestamp() {
+        return timestamp;
+    }
+
+    /**
+     * Bucketspan expressed in seconds
+     */
+    public long getBucketSpan() {
+        return bucketSpan;
+    }
+
+    public double getAnomalyScore() {
+        return anomalyScore;
+    }
+
+    void setAnomalyScore(double anomalyScore) {
+        this.anomalyScore = anomalyScore;
+    }
+
+    public double getInitialAnomalyScore() {
+        return initialAnomalyScore;
+    }
+
+    void setInitialAnomalyScore(double initialAnomalyScore) {
+        this.initialAnomalyScore = initialAnomalyScore;
+    }
+
+    /**
+     * Get all the anomaly records associated with this bucket.
+     * The records are not part of the bucket document. They will
+     * only be present when the bucket was retrieved and expanded
+     * to contain the associated records.
+     *
+     * @return the anomaly records for the bucket IF the bucket was expanded.
+     */
+    public List<AnomalyRecord> getRecords() {
+        return records;
+    }
+
+    void setRecords(List<AnomalyRecord> records) {
+        this.records = Collections.unmodifiableList(records);
+    }
+
+    /**
+     * The number of records (events) actually processed in this bucket.
+     */
+    public long getEventCount() {
+        return eventCount;
+    }
+
+    void setEventCount(long value) {
+        eventCount = value;
+    }
+
+    public boolean isInterim() {
+        return isInterim;
+    }
+
+    void setInterim(boolean isInterim) {
+        this.isInterim = isInterim;
+    }
+
+    public long getProcessingTimeMs() {
+        return processingTimeMs;
+    }
+
+    void setProcessingTimeMs(long timeMs) {
+        processingTimeMs = timeMs;
+    }
+
+    public List<BucketInfluencer> getBucketInfluencers() {
+        return bucketInfluencers;
+    }
+
+    void setBucketInfluencers(List<BucketInfluencer> bucketInfluencers) {
+        this.bucketInfluencers = Collections.unmodifiableList(bucketInfluencers);
+    }
+
+    public List<String> getScheduledEvents() {
+        return scheduledEvents;
+    }
+
+    void setScheduledEvents(List<String> scheduledEvents) {
+        this.scheduledEvents = Collections.unmodifiableList(scheduledEvents);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, timestamp, eventCount, initialAnomalyScore, anomalyScore, records,
+                isInterim, bucketSpan, bucketInfluencers, processingTimeMs, scheduledEvents);
+    }
+
+    /**
+     * Compare all the fields and embedded anomaly records (if any)
+     */
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        Bucket that = (Bucket) other;
+
+        return Objects.equals(this.jobId, that.jobId) && Objects.equals(this.timestamp, that.timestamp)
+                && (this.eventCount == that.eventCount) && (this.bucketSpan == that.bucketSpan)
+                && (this.anomalyScore == that.anomalyScore) && (this.initialAnomalyScore == that.initialAnomalyScore)
+                && Objects.equals(this.records, that.records) && Objects.equals(this.isInterim, that.isInterim)
+                && Objects.equals(this.bucketInfluencers, that.bucketInfluencers)
+                && (this.processingTimeMs == that.processingTimeMs)
+                && Objects.equals(this.scheduledEvents, that.scheduledEvents);
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketInfluencer.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketInfluencer.java
new file mode 100644
index 0000000000000..29d8447cd6a37
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketInfluencer.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.io.IOException;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
+import java.util.Objects;
+
+public class BucketInfluencer implements ToXContentObject {
+
+    /**
+     * Result type
+     */
+    public static final String RESULT_TYPE_VALUE = "bucket_influencer";
+    public static final ParseField RESULT_TYPE_FIELD = new ParseField(RESULT_TYPE_VALUE);
+
+    /**
+     * Field names
+     */
+    public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencer_field_name");
+    public static final ParseField INITIAL_ANOMALY_SCORE = new ParseField("initial_anomaly_score");
+    public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score");
+    public static final ParseField RAW_ANOMALY_SCORE = new ParseField("raw_anomaly_score");
+    public static final ParseField PROBABILITY = new ParseField("probability");
+    public static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
+
+    public static final ConstructingObjectParser<BucketInfluencer, Void> PARSER =
+        new ConstructingObjectParser<>(RESULT_TYPE_FIELD.getPreferredName(), true,
+            a -> new BucketInfluencer((String) a[0], (Date) a[1], (long) a[2]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
+        PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> {
+            if (p.currentToken() == Token.VALUE_NUMBER) {
+                return new Date(p.longValue());
+            } else if (p.currentToken() == Token.VALUE_STRING) {
+                return new Date(DateFormatters.toZonedDateTime(DateTimeFormatter.ISO_INSTANT.parse(p.text())).toInstant().toEpochMilli());
+            }
+            throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for ["
+                    + Result.TIMESTAMP.getPreferredName() + "]");
+        }, Result.TIMESTAMP, ValueType.VALUE);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN);
+        PARSER.declareString((bucketInfluencer, s) -> {}, Result.RESULT_TYPE);
+        PARSER.declareString(BucketInfluencer::setInfluencerFieldName, INFLUENCER_FIELD_NAME);
+        PARSER.declareDouble(BucketInfluencer::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE);
+        PARSER.declareDouble(BucketInfluencer::setAnomalyScore, ANOMALY_SCORE);
+        PARSER.declareDouble(BucketInfluencer::setRawAnomalyScore, RAW_ANOMALY_SCORE);
+        PARSER.declareDouble(BucketInfluencer::setProbability, PROBABILITY);
+        PARSER.declareBoolean(BucketInfluencer::setIsInterim, Result.IS_INTERIM);
+    }
+
+    private final String jobId;
+    private String influenceField;
+    private double initialAnomalyScore;
+    private double anomalyScore;
+    private double rawAnomalyScore;
+    private double probability;
+    private boolean isInterim;
+    private final Date timestamp;
+    private final long bucketSpan;
+
+    BucketInfluencer(String jobId, Date timestamp, long bucketSpan) {
+        this.jobId = jobId;
+        this.timestamp = Objects.requireNonNull(timestamp);
+        this.bucketSpan = bucketSpan;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Job.ID.getPreferredName(), jobId);
+        builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
+        if (influenceField != null) {
+            builder.field(INFLUENCER_FIELD_NAME.getPreferredName(), influenceField);
+        }
+        builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore);
+        builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore);
+        builder.field(RAW_ANOMALY_SCORE.getPreferredName(), rawAnomalyScore);
+        builder.field(PROBABILITY.getPreferredName(), probability);
+        builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
+        builder.field(Result.IS_INTERIM.getPreferredName(), isInterim);
+        builder.endObject();
+        return builder;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public double getProbability() {
+        return probability;
+    }
+
+    void setProbability(double probability) {
+        this.probability = probability;
+    }
+
+    public String getInfluencerFieldName() {
+        return influenceField;
+    }
+
+    void setInfluencerFieldName(String fieldName) {
+        this.influenceField = fieldName;
+    }
+
+    public double getInitialAnomalyScore() {
+        return initialAnomalyScore;
+    }
+
+    void setInitialAnomalyScore(double influenceScore) {
+        this.initialAnomalyScore = influenceScore;
+    }
+
+    public double getAnomalyScore() {
+        return anomalyScore;
+    }
+
+    void setAnomalyScore(double score) {
+        anomalyScore = score;
+    }
+
+    public double getRawAnomalyScore() {
+        return rawAnomalyScore;
+    }
+
+    void setRawAnomalyScore(double score) {
+        rawAnomalyScore = score;
+    }
+
+    void setIsInterim(boolean isInterim) {
+        this.isInterim = isInterim;
+    }
+
+    public boolean isInterim() {
+        return isInterim;
+    }
+
+    public Date getTimestamp() {
+        return timestamp;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(influenceField, initialAnomalyScore, anomalyScore, rawAnomalyScore, probability, isInterim, timestamp, jobId,
+                bucketSpan);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null) {
+            return false;
+        }
+
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+
+        BucketInfluencer other = (BucketInfluencer) obj;
+
+        return Objects.equals(influenceField, other.influenceField) && Double.compare(initialAnomalyScore, other.initialAnomalyScore) == 0
+                && Double.compare(anomalyScore, other.anomalyScore) == 0 && Double.compare(rawAnomalyScore, other.rawAnomalyScore) == 0
+                && Double.compare(probability, other.probability) == 0 && Objects.equals(isInterim, other.isInterim)
+                && Objects.equals(timestamp, other.timestamp) && Objects.equals(jobId, other.jobId) && bucketSpan == other.bucketSpan;
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/CategoryDefinition.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/CategoryDefinition.java
new file mode 100644
index 0000000000000..59b59006b33a1
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/CategoryDefinition.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+
+public class CategoryDefinition implements ToXContentObject {
+
+    public static final ParseField TYPE = new ParseField("category_definition");
+
+    public static final ParseField CATEGORY_ID = new ParseField("category_id");
+    public static final ParseField TERMS = new ParseField("terms");
+    public static final ParseField REGEX = new ParseField("regex");
+    public static final ParseField MAX_MATCHING_LENGTH = new ParseField("max_matching_length");
+    public static final ParseField EXAMPLES = new ParseField("examples");
+    public static final ParseField GROK_PATTERN = new ParseField("grok_pattern");
+
+    // Used for QueryPage
+    public static final ParseField RESULTS_FIELD = new ParseField("categories");
+
+    public static final ConstructingObjectParser<CategoryDefinition, Void> PARSER =
+        new ConstructingObjectParser<>(TYPE.getPreferredName(), true, a -> new CategoryDefinition((String) a[0]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
+        PARSER.declareLong(CategoryDefinition::setCategoryId, CATEGORY_ID);
+        PARSER.declareString(CategoryDefinition::setTerms, TERMS);
+        PARSER.declareString(CategoryDefinition::setRegex, REGEX);
+        PARSER.declareLong(CategoryDefinition::setMaxMatchingLength, MAX_MATCHING_LENGTH);
+        PARSER.declareStringArray(CategoryDefinition::setExamples, EXAMPLES);
+        PARSER.declareString(CategoryDefinition::setGrokPattern, GROK_PATTERN);
+    }
+
+    private final String jobId;
+    private long categoryId = 0L;
+    private String terms = "";
+    private String regex = "";
+    private long maxMatchingLength = 0L;
+    private final Set<String> examples = new TreeSet<>();
+    private String grokPattern;
+
+    CategoryDefinition(String jobId) {
+        this.jobId = jobId;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public long getCategoryId() {
+        return categoryId;
+    }
+
+    void setCategoryId(long categoryId) {
+        this.categoryId = categoryId;
+    }
+
+    public String getTerms() {
+        return terms;
+    }
+
+    void setTerms(String terms) {
+        this.terms = terms;
+    }
+
+    public String getRegex() {
+        return regex;
+    }
+
+    void setRegex(String regex) {
+        this.regex = regex;
+    }
+
+    public long getMaxMatchingLength() {
+        return maxMatchingLength;
+    }
+
+    void setMaxMatchingLength(long maxMatchingLength) {
+        this.maxMatchingLength = maxMatchingLength;
+    }
+
+    public List<String> getExamples() {
+        return new ArrayList<>(examples);
+    }
+
+    void setExamples(Collection<String> examples) {
+        this.examples.clear();
+        this.examples.addAll(examples);
+    }
+
+    void addExample(String example) {
+        examples.add(example);
+    }
+
+    public String getGrokPattern() {
+        return grokPattern;
+    }
+
+    void setGrokPattern(String grokPattern) {
+        this.grokPattern = grokPattern;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Job.ID.getPreferredName(), jobId);
+        builder.field(CATEGORY_ID.getPreferredName(), categoryId);
+        builder.field(TERMS.getPreferredName(), terms);
+        builder.field(REGEX.getPreferredName(), regex);
+        builder.field(MAX_MATCHING_LENGTH.getPreferredName(), maxMatchingLength);
+        builder.field(EXAMPLES.getPreferredName(), examples);
+        if (grokPattern != null) {
+            builder.field(GROK_PATTERN.getPreferredName(), grokPattern);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+        CategoryDefinition that = (CategoryDefinition) other;
+        return Objects.equals(this.jobId, that.jobId)
+                && Objects.equals(this.categoryId, that.categoryId)
+                && Objects.equals(this.terms, that.terms)
+                && Objects.equals(this.regex, that.regex)
+                && Objects.equals(this.maxMatchingLength, that.maxMatchingLength)
+                && Objects.equals(this.examples, that.examples)
+                && Objects.equals(this.grokPattern, that.grokPattern);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, categoryId, terms, regex, maxMatchingLength, examples, grokPattern);
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Influence.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Influence.java
new file mode 100644
index 0000000000000..53607479d66f4
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Influence.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Influence field name and list of influence field values/score pairs
+ */
+public class Influence implements ToXContentObject {
+
+    /**
+     * Note all X-Content serialized field names are "influencer" not "influence"
+     */
+    public static final ParseField INFLUENCER = new ParseField("influencer");
+    public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencer_field_name");
+    public static final ParseField INFLUENCER_FIELD_VALUES = new ParseField("influencer_field_values");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<Influence, Void> PARSER =
+        new ConstructingObjectParser<>(INFLUENCER.getPreferredName(), true, a -> new Influence((String) a[0], (List<String>) a[1]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_NAME);
+        PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_VALUES);
+    }
+
+    private String field;
+    private List<String> fieldValues;
+
+    Influence(String field, List<String> fieldValues) {
+        this.field = field;
+        this.fieldValues = Collections.unmodifiableList(fieldValues);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(INFLUENCER_FIELD_NAME.getPreferredName(), field);
+        builder.field(INFLUENCER_FIELD_VALUES.getPreferredName(), fieldValues);
+        builder.endObject();
+        return builder;
+    }
+
+    public String getInfluencerFieldName() {
+        return field;
+    }
+
+    public List<String> getInfluencerFieldValues() {
+        return fieldValues;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field, fieldValues);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null) {
+            return false;
+        }
+
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+
+        Influence other = (Influence) obj;
+        return Objects.equals(field, other.field) && Objects.equals(fieldValues, other.fieldValues);
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Influencer.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Influencer.java
new file mode 100644
index 0000000000000..51c88883608b0
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Influencer.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.io.IOException;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
+import java.util.Objects;
+
+public class Influencer implements ToXContentObject {
+
+    /**
+     * Result type
+     */
+    public static final String RESULT_TYPE_VALUE = "influencer";
+    public static final ParseField RESULT_TYPE_FIELD = new ParseField(RESULT_TYPE_VALUE);
+
+    /*
+     * Field names
+     */
+    public static final ParseField PROBABILITY = new ParseField("probability");
+    public static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
+    public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencer_field_name");
+    public static final ParseField INFLUENCER_FIELD_VALUE = new ParseField("influencer_field_value");
+    public static final ParseField INITIAL_INFLUENCER_SCORE = new ParseField("initial_influencer_score");
+    public static final ParseField INFLUENCER_SCORE = new ParseField("influencer_score");
+
+    // Used for QueryPage
+    public static final ParseField RESULTS_FIELD = new ParseField("influencers");
+
+    public static final ConstructingObjectParser<Influencer, Void> PARSER = new ConstructingObjectParser<>(
+            RESULT_TYPE_FIELD.getPreferredName(), true,
+            a -> new Influencer((String) a[0], (String) a[1], (String) a[2], (Date) a[3], (long) a[4]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_NAME);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_VALUE);
+        PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> {
+            if (p.currentToken() == Token.VALUE_NUMBER) {
+                return new Date(p.longValue());
+            } else if (p.currentToken() == Token.VALUE_STRING) {
+                return new Date(DateFormatters.toZonedDateTime(DateTimeFormatter.ISO_INSTANT.parse(p.text())).toInstant().toEpochMilli());
+            }
+            throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for ["
+                    + Result.TIMESTAMP.getPreferredName() + "]");
+        }, Result.TIMESTAMP, ValueType.VALUE);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN);
+        PARSER.declareString((influencer, s) -> {}, Result.RESULT_TYPE);
+        PARSER.declareDouble(Influencer::setProbability, PROBABILITY);
+        PARSER.declareDouble(Influencer::setInfluencerScore, INFLUENCER_SCORE);
+        PARSER.declareDouble(Influencer::setInitialInfluencerScore, INITIAL_INFLUENCER_SCORE);
+        PARSER.declareBoolean(Influencer::setInterim, Result.IS_INTERIM);
+    }
+
+    private final String jobId;
+    private final Date timestamp;
+    private final long bucketSpan;
+    private String influenceField;
+    private String influenceValue;
+    private double probability;
+    private double initialInfluencerScore;
+    private double influencerScore;
+    private boolean isInterim;
+
+    Influencer(String jobId, String fieldName, String fieldValue, Date timestamp, long bucketSpan) {
+        this.jobId = jobId;
+        influenceField = fieldName;
+        influenceValue = fieldValue;
+        this.timestamp = Objects.requireNonNull(timestamp);
+        this.bucketSpan = bucketSpan;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Job.ID.getPreferredName(), jobId);
+        builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
+        builder.field(INFLUENCER_FIELD_NAME.getPreferredName(), influenceField);
+        builder.field(INFLUENCER_FIELD_VALUE.getPreferredName(), influenceValue);
+        builder.field(INFLUENCER_SCORE.getPreferredName(), influencerScore);
+        builder.field(INITIAL_INFLUENCER_SCORE.getPreferredName(), initialInfluencerScore);
+        builder.field(PROBABILITY.getPreferredName(), probability);
+        builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
+        builder.field(Result.IS_INTERIM.getPreferredName(), isInterim);
+        builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        builder.endObject();
+        return builder;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public double getProbability() {
+        return probability;
+    }
+
+    void setProbability(double probability) {
+        this.probability = probability;
+    }
+
+    public Date getTimestamp() {
+        return timestamp;
+    }
+
+    public String getInfluencerFieldName() {
+        return influenceField;
+    }
+
+    public String getInfluencerFieldValue() {
+        return influenceValue;
+    }
+
+    public double getInitialInfluencerScore() {
+        return initialInfluencerScore;
+    }
+
+    void setInitialInfluencerScore(double score) {
+        initialInfluencerScore = score;
+    }
+
+    public double getInfluencerScore() {
+        return influencerScore;
+    }
+
+    void setInfluencerScore(double score) {
+        influencerScore = score;
+    }
+
+    public boolean isInterim() {
+        return isInterim;
+    }
+
+    void setInterim(boolean value) {
+        isInterim = value;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, timestamp, influenceField, influenceValue, initialInfluencerScore,
+                influencerScore, probability, isInterim, bucketSpan);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null) {
+            return false;
+        }
+
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+
+        Influencer other = (Influencer) obj;
+        return Objects.equals(jobId, other.jobId) && Objects.equals(timestamp, other.timestamp)
+                && Objects.equals(influenceField, other.influenceField)
+                && Objects.equals(influenceValue, other.influenceValue)
+                && Double.compare(initialInfluencerScore, other.initialInfluencerScore) == 0
+                && Double.compare(influencerScore, other.influencerScore) == 0 && Double.compare(probability, other.probability) == 0
+                && (isInterim == other.isInterim) && (bucketSpan == other.bucketSpan);
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/OverallBucket.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/OverallBucket.java
new file mode 100644
index 0000000000000..4f13b4b26646e
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/OverallBucket.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.io.IOException;
+import java.time.format.DateTimeFormatter;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Overall Bucket Result POJO
+ */
+public class OverallBucket implements ToXContentObject {
+
+    public static final ParseField OVERALL_SCORE = new ParseField("overall_score");
+    public static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
+    public static final ParseField JOBS = new ParseField("jobs");
+
+    // Used for QueryPage
+    public static final ParseField RESULTS_FIELD = new ParseField("overall_buckets");
+
+    /**
+     * Result type
+     */
+    public static final String RESULT_TYPE_VALUE = "overall_bucket";
+
+    public static final ConstructingObjectParser<OverallBucket, Void> PARSER =
+        new ConstructingObjectParser<>(RESULT_TYPE_VALUE, true,
+            a -> new OverallBucket((Date) a[0], (long) a[1], (double) a[2], (boolean) a[3]));
+
+    static {
+        PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+                return new Date(p.longValue());
+            } else if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return new Date(DateFormatters.toZonedDateTime(DateTimeFormatter.ISO_INSTANT.parse(p.text())).toInstant().toEpochMilli());
+            }
+            throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for ["
+                + Result.TIMESTAMP.getPreferredName() + "]");
+        }, Result.TIMESTAMP, ObjectParser.ValueType.VALUE);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN);
+        PARSER.declareDouble(ConstructingObjectParser.constructorArg(), OVERALL_SCORE);
+        PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), Result.IS_INTERIM);
+        PARSER.declareObjectArray(OverallBucket::setJobs, JobInfo.PARSER, JOBS);
+    }
+
+    private final Date timestamp;
+    private final long bucketSpan;
+    private final double overallScore;
+    private final boolean isInterim;
+    private List<JobInfo> jobs = Collections.emptyList();
+
+    OverallBucket(Date timestamp, long bucketSpan, double overallScore, boolean isInterim) {
+        this.timestamp = Objects.requireNonNull(timestamp);
+        this.bucketSpan = bucketSpan;
+        this.overallScore = overallScore;
+        this.isInterim = isInterim;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
+        builder.field(OVERALL_SCORE.getPreferredName(), overallScore);
+        builder.field(JOBS.getPreferredName(), jobs);
+        builder.field(Result.IS_INTERIM.getPreferredName(), isInterim);
+        builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
+        builder.endObject();
+        return builder;
+    }
+
+    public Date getTimestamp() {
+        return timestamp;
+    }
+
+    /**
+     * Bucketspan expressed in seconds
+     */
+    public long getBucketSpan() {
+        return bucketSpan;
+    }
+
+    public double getOverallScore() {
+        return overallScore;
+    }
+
+    public List<JobInfo> getJobs() {
+        return jobs;
+    }
+
+    void setJobs(List<JobInfo> jobs) {
+        this.jobs = Collections.unmodifiableList(jobs);
+    }
+
+    public boolean isInterim() {
+        return isInterim;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(timestamp, bucketSpan, overallScore, jobs, isInterim);
+    }
+
+    /**
+     * Compare all the fields and embedded anomaly records (if any)
+     */
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        OverallBucket that = (OverallBucket) other;
+
+        return Objects.equals(this.timestamp, that.timestamp)
+                && this.bucketSpan == that.bucketSpan
+                && this.overallScore == that.overallScore
+                && Objects.equals(this.jobs, that.jobs)
+                && this.isInterim == that.isInterim;
+    }
+
+    public static class JobInfo implements ToXContentObject, Comparable<JobInfo> {
+
+        private static final ParseField MAX_ANOMALY_SCORE = new ParseField("max_anomaly_score");
+
+        public static final ConstructingObjectParser<JobInfo, Void> PARSER =
+            new ConstructingObjectParser<>("job_info", true, a -> new JobInfo((String) a[0], (double) a[1]));
+
+        static {
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
+            PARSER.declareDouble(ConstructingObjectParser.constructorArg(), MAX_ANOMALY_SCORE);
+        }
+
+        private final String jobId;
+        private final double maxAnomalyScore;
+
+        JobInfo(String jobId, double maxAnomalyScore) {
+            this.jobId = Objects.requireNonNull(jobId);
+            this.maxAnomalyScore = maxAnomalyScore;
+        }
+
+        public String getJobId() {
+            return jobId;
+        }
+
+        public double getMaxAnomalyScore() {
+            return maxAnomalyScore;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(Job.ID.getPreferredName(), jobId);
+            builder.field(MAX_ANOMALY_SCORE.getPreferredName(), maxAnomalyScore);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(jobId, maxAnomalyScore);
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (this == other) {
+                return true;
+            }
+            if (other == null || getClass() != other.getClass()) {
+                return false;
+            }
+            JobInfo that = (JobInfo) other;
+            return Objects.equals(this.jobId, that.jobId) && this.maxAnomalyScore == that.maxAnomalyScore;
+        }
+
+        @Override
+        public int compareTo(JobInfo other) {
+            int result = this.jobId.compareTo(other.jobId);
+            if (result == 0) {
+                result = Double.compare(this.maxAnomalyScore, other.maxAnomalyScore);
+            }
+            return result;
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Result.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Result.java
new file mode 100644
index 0000000000000..cce5fa65ebb44
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Result.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.ParseField;
+
+/**
+ * Contains common attributes for results.
+ */
+public final class Result {
+
+    /**
+     * Serialisation fields
+     */
+    public static final ParseField TYPE = new ParseField("result");
+    public static final ParseField RESULT_TYPE = new ParseField("result_type");
+    public static final ParseField TIMESTAMP = new ParseField("timestamp");
+    public static final ParseField IS_INTERIM = new ParseField("is_interim");
+
+    private Result() {
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/util/TimeUtil.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/util/TimeUtil.java
new file mode 100644
index 0000000000000..549b196949145
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/util/TimeUtil.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.util;
+
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
+
+public final class TimeUtil {
+
+    /**
+     * Parse out a Date object given the current parser and field name.
+     *
+     * @param parser current XContentParser
+     * @param fieldName the field's preferred name (utilized in exception)
+     * @return parsed Date object
+     * @throws IOException from XContentParser
+     */
+    public static Date parseTimeField(XContentParser parser, String fieldName) throws IOException {
+        if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+            return new Date(parser.longValue());
+        } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
+            return new Date(DateFormatters.toZonedDateTime(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli());
+        }
+        throw new IllegalArgumentException(
+            "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
+    }
+
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/package-info.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/package-info.java
new file mode 100644
index 0000000000000..b1e4c6c0d4e54
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Request and Response objects for the default distribution's Machine
+ * Learning APIs.
+ */
+package org.elasticsearch.protocol.xpack.ml;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/security/User.java
similarity index 91%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java
rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/security/User.java
index e8161b9a7e239..42e957ecf2d51 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/security/User.java
@@ -1,9 +1,23 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
-package org.elasticsearch.xpack.core.security.user;
+
+package org.elasticsearch.protocol.xpack.security;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
@@ -13,7 +27,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -128,7 +141,7 @@ public String toString() {
         sb.append(",fullName=").append(fullName);
         sb.append(",email=").append(email);
         sb.append(",metadata=");
-        MetadataUtils.writeValue(sb, metadata);
+        sb.append(metadata);
         if (authenticatedUser != null) {
             sb.append(",authenticatedUser=[").append(authenticatedUser.toString()).append("]");
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchRequest.java
similarity index 67%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequest.java
rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchRequest.java
index f8c1a71f1ebc7..1ec83a8c05ae7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequest.java
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchRequest.java
@@ -1,9 +1,22 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
-package org.elasticsearch.xpack.core.watcher.transport.actions.delete;
+package org.elasticsearch.protocol.xpack.watcher;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -11,7 +24,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.uid.Versions;
-import org.elasticsearch.xpack.core.watcher.support.WatcherUtils;
 
 import java.io.IOException;
 
@@ -50,7 +62,7 @@ public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
         if (id == null){
             validationException = ValidateActions.addValidationError("watch id is missing", validationException);
-        } else if (WatcherUtils.isValidId(id) == false) {
+        } else if (PutWatchRequest.isValidId(id) == false) {
             validationException = ValidateActions.addValidationError("watch id contains whitespace", validationException);
         }
         return validationException;
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponse.java
new file mode 100644
index 0000000000000..b644a6a854cfa
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponse.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.watcher;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class DeleteWatchResponse extends ActionResponse implements ToXContentObject {
+
+    private static final ObjectParser<DeleteWatchResponse, Void> PARSER
+        = new ObjectParser<>("x_pack_delete_watch_response", DeleteWatchResponse::new);
+    static {
+        PARSER.declareString(DeleteWatchResponse::setId, new ParseField("_id"));
+        PARSER.declareLong(DeleteWatchResponse::setVersion, new ParseField("_version"));
+        PARSER.declareBoolean(DeleteWatchResponse::setFound, new ParseField("found"));
+    }
+
+    private String id;
+    private long version;
+    private boolean found;
+
+    public DeleteWatchResponse() {
+    }
+
+    public DeleteWatchResponse(String id, long version, boolean found) {
+        this.id = id;
+        this.version = version;
+        this.found = found;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public long getVersion() {
+        return version;
+    }
+
+    public boolean isFound() {
+        return found;
+    }
+
+    private void setId(String id) {
+        this.id = id;
+    }
+
+    private void setVersion(long version) {
+        this.version = version;
+    }
+
+    private void setFound(boolean found) {
+        this.found = found;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        DeleteWatchResponse that = (DeleteWatchResponse) o;
+
+        return Objects.equals(id, that.id) && Objects.equals(version, that.version) && Objects.equals(found, that.found);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, version, found);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        id = in.readString();
+        version = in.readVLong();
+        found = in.readBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(id);
+        out.writeVLong(version);
+        out.writeBoolean(found);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.startObject()
+            .field("_id", id)
+            .field("_version", version)
+            .field("found", found)
+            .endObject();
+    }
+
+    public static DeleteWatchResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/DeleteLicenseResponseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/DeleteLicenseResponseTests.java
new file mode 100644
index 0000000000000..f4caa1f42421d
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/DeleteLicenseResponseTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.license;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+
+public class DeleteLicenseResponseTests extends AbstractStreamableXContentTestCase<DeleteLicenseResponse> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected DeleteLicenseResponse createTestInstance() {
+        return new DeleteLicenseResponse(randomBoolean());
+    }
+
+    @Override
+    protected DeleteLicenseResponse doParseInstance(XContentParser parser) {
+        return DeleteLicenseResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected DeleteLicenseResponse createBlankInstance() {
+        return new DeleteLicenseResponse();
+    }
+
+    @Override
+    protected DeleteLicenseResponse mutateInstance(DeleteLicenseResponse response) {
+        return new DeleteLicenseResponse(!response.isAcknowledged());
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoRequestTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoRequestTests.java
new file mode 100644
index 0000000000000..a4daa39566462
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoRequestTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.protocol.xpack.migration;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+public class IndexUpgradeInfoRequestTests extends AbstractWireSerializingTestCase<IndexUpgradeInfoRequest> {
+    @Override
+    protected IndexUpgradeInfoRequest createTestInstance() {
+        int indexCount = randomInt(4);
+        String[] indices = new String[indexCount];
+        for (int i = 0; i < indexCount; i++) {
+            indices[i] = randomAlphaOfLength(10);
+        }
+        IndexUpgradeInfoRequest request = new IndexUpgradeInfoRequest(indices);
+        if (randomBoolean()) {
+            request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+        }
+        return request;
+    }
+
+    @Override
+    protected Writeable.Reader<IndexUpgradeInfoRequest> instanceReader() {
+        return IndexUpgradeInfoRequest::new;
+    }
+
+    public void testNullIndices() {
+        expectThrows(NullPointerException.class, () -> new IndexUpgradeInfoRequest((String[])null));
+        expectThrows(NullPointerException.class, () -> new IndexUpgradeInfoRequest().indices((String[])null));
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoResponseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoResponseTests.java
new file mode 100644
index 0000000000000..42de1ae60908a
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoResponseTests.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.protocol.xpack.migration;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+public class IndexUpgradeInfoResponseTests extends AbstractStreamableXContentTestCase<IndexUpgradeInfoResponse> {
+    @Override
+    protected IndexUpgradeInfoResponse doParseInstance(XContentParser parser) {
+        return IndexUpgradeInfoResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected IndexUpgradeInfoResponse createBlankInstance() {
+        return new IndexUpgradeInfoResponse();
+    }
+
+    @Override
+    protected IndexUpgradeInfoResponse createTestInstance() {
+        return randomIndexUpgradeInfoResponse(randomIntBetween(0, 10));
+    }
+
+    private static IndexUpgradeInfoResponse randomIndexUpgradeInfoResponse(int numIndices) {
+        Map<String, UpgradeActionRequired> actions = new HashMap<>();
+        for (int i = 0; i < numIndices; i++) {
+            actions.put(randomAlphaOfLength(5), randomFrom(UpgradeActionRequired.values()));
+        }
+        return new IndexUpgradeInfoResponse(actions);
+    }
+
+    @Override
+    protected IndexUpgradeInfoResponse mutateInstance(IndexUpgradeInfoResponse instance) {
+        if (instance.getActions().size() == 0) {
+            return randomIndexUpgradeInfoResponse(1);
+        }
+        Map<String, UpgradeActionRequired> actions = new HashMap<>(instance.getActions());
+        if (randomBoolean()) {
+            Iterator<Map.Entry<String, UpgradeActionRequired>> iterator = actions.entrySet().iterator();
+            iterator.next();
+            iterator.remove();
+        } else {
+            actions.put(randomAlphaOfLength(5), randomFrom(UpgradeActionRequired.values()));
+        }
+        return new IndexUpgradeInfoResponse(actions);
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/PutJobRequestTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/PutJobRequestTests.java
new file mode 100644
index 0000000000000..448c40a4d2fa1
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/PutJobRequestTests.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+import org.elasticsearch.protocol.xpack.ml.job.config.JobTests;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class PutJobRequestTests extends AbstractXContentTestCase<PutJobRequest> {
+
+    @Override
+    protected PutJobRequest createTestInstance() {
+        return new PutJobRequest(JobTests.createRandomizedJob());
+    }
+
+    @Override
+    protected PutJobRequest doParseInstance(XContentParser parser) throws IOException {
+        return new PutJobRequest(Job.PARSER.apply(parser, null).build());
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/PutJobResponseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/PutJobResponseTests.java
new file mode 100644
index 0000000000000..ed91e33635b29
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/PutJobResponseTests.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.protocol.xpack.ml.job.config.JobTests;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class PutJobResponseTests extends AbstractXContentTestCase<PutJobResponse> {
+
+    @Override
+    protected PutJobResponse createTestInstance() {
+        return new PutJobResponse(JobTests.createRandomizedJob());
+    }
+
+    @Override
+    protected PutJobResponse doParseInstance(XContentParser parser) throws IOException {
+        return PutJobResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/datafeed/ChunkingConfigTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/datafeed/ChunkingConfigTests.java
new file mode 100644
index 0000000000000..c835788bb1c98
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/datafeed/ChunkingConfigTests.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.datafeed;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+public class ChunkingConfigTests extends AbstractXContentTestCase<ChunkingConfig> {
+
+    @Override
+    protected ChunkingConfig createTestInstance() {
+        return createRandomizedChunk();
+    }
+
+    @Override
+    protected ChunkingConfig doParseInstance(XContentParser parser) {
+        return ChunkingConfig.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    public static ChunkingConfig createRandomizedChunk() {
+        ChunkingConfig.Mode mode = randomFrom(ChunkingConfig.Mode.values());
+        TimeValue timeSpan = null;
+        if (mode == ChunkingConfig.Mode.MANUAL) {
+            // time span is required to be at least 1 millis, so we use a custom method to generate a time value here
+            timeSpan = randomPositiveSecondsMinutesHours();
+        }
+        return new ChunkingConfig(mode, timeSpan);
+    }
+
+    private static TimeValue randomPositiveSecondsMinutesHours() {
+        return new TimeValue(randomIntBetween(1, 1000), randomFrom(Arrays.asList(TimeUnit.SECONDS, TimeUnit.MINUTES, TimeUnit.HOURS)));
+    }
+
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedConfigTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedConfigTests.java
new file mode 100644
index 0000000000000..f45d88d318e01
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedConfigTests.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.datafeed;
+
+import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class DatafeedConfigTests extends AbstractXContentTestCase<DatafeedConfig> {
+
+    @Override
+    protected DatafeedConfig createTestInstance() {
+        long bucketSpanMillis = 3600000;
+        DatafeedConfig.Builder builder = constructBuilder();
+        builder.setIndices(randomStringList(1, 10));
+        builder.setTypes(randomStringList(0, 10));
+        if (randomBoolean()) {
+            builder.setQuery(QueryBuilders.termQuery(randomAlphaOfLength(10), randomAlphaOfLength(10)));
+        }
+        boolean addScriptFields = randomBoolean();
+        if (addScriptFields) {
+            int scriptsSize = randomInt(3);
+            List<ScriptField> scriptFields = new ArrayList<>(scriptsSize);
+            for (int scriptIndex = 0; scriptIndex < scriptsSize; scriptIndex++) {
+                scriptFields.add(new ScriptField(randomAlphaOfLength(10), mockScript(randomAlphaOfLength(10)),
+                    randomBoolean()));
+            }
+            builder.setScriptFields(scriptFields);
+        }
+        Long aggHistogramInterval = null;
+        if (randomBoolean()) {
+            // can only test with a single agg as the xcontent order gets randomized by test base class and then
+            // the actual xcontent isn't the same and test fail.
+            // Testing with a single agg is ok as we don't have special list xcontent logic
+            AggregatorFactories.Builder aggs = new AggregatorFactories.Builder();
+            aggHistogramInterval = randomNonNegativeLong();
+            aggHistogramInterval = aggHistogramInterval > bucketSpanMillis ? bucketSpanMillis : aggHistogramInterval;
+            aggHistogramInterval = aggHistogramInterval <= 0 ? 1 : aggHistogramInterval;
+            MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
+            aggs.addAggregator(AggregationBuilders.dateHistogram("buckets")
+                .interval(aggHistogramInterval).subAggregation(maxTime).field("time"));
+            builder.setAggregations(aggs);
+        }
+        if (randomBoolean()) {
+            builder.setScrollSize(randomIntBetween(0, Integer.MAX_VALUE));
+        }
+        if (randomBoolean()) {
+            if (aggHistogramInterval == null) {
+                builder.setFrequency(TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000)));
+            } else {
+                builder.setFrequency(TimeValue.timeValueMillis(randomIntBetween(1, 5) * aggHistogramInterval));
+            }
+        }
+        if (randomBoolean()) {
+            builder.setQueryDelay(TimeValue.timeValueMillis(randomIntBetween(1, 1_000_000)));
+        }
+        if (randomBoolean()) {
+            builder.setChunkingConfig(ChunkingConfigTests.createRandomizedChunk());
+        }
+        return builder.build();
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        return new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+
+    public static List<String> randomStringList(int min, int max) {
+        int size = scaledRandomIntBetween(min, max);
+        List<String> list = new ArrayList<>();
+        for (int i = 0; i < size; i++) {
+            list.add(randomAlphaOfLength(10));
+        }
+        return list;
+    }
+
+    @Override
+    protected DatafeedConfig doParseInstance(XContentParser parser) {
+        return DatafeedConfig.PARSER.apply(parser, null).build();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    private static final String FUTURE_DATAFEED = "{\n" +
+        "    \"datafeed_id\": \"farequote-datafeed\",\n" +
+        "    \"job_id\": \"farequote\",\n" +
+        "    \"frequency\": \"1h\",\n" +
+        "    \"indices\": [\"farequote1\", \"farequote2\"],\n" +
+        "    \"tomorrows_technology_today\": \"amazing\",\n" +
+        "    \"scroll_size\": 1234\n" +
+        "}";
+
+    public void testFutureMetadataParse() throws IOException {
+        XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, FUTURE_DATAFEED);
+        // Unlike the config version of this test, the metadata parser should tolerate the unknown future field
+        assertNotNull(DatafeedConfig.PARSER.apply(parser, null).build());
+    }
+
+    public void testCopyConstructor() {
+        for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
+            DatafeedConfig datafeedConfig = createTestInstance();
+            DatafeedConfig copy = new DatafeedConfig.Builder(datafeedConfig).build();
+            assertEquals(datafeedConfig, copy);
+        }
+    }
+
+    public void testCheckValid_GivenNullIdInConstruction() {
+        expectThrows(NullPointerException.class, () -> new DatafeedConfig.Builder(null, null));
+    }
+
+    public void testCheckValid_GivenNullJobId() {
+        expectThrows(NullPointerException.class, () -> new DatafeedConfig.Builder(randomValidDatafeedId(), null));
+    }
+
+    public void testCheckValid_GivenNullIndices() {
+        DatafeedConfig.Builder conf = constructBuilder();
+        expectThrows(NullPointerException.class, () -> conf.setIndices(null));
+    }
+
+    public void testCheckValid_GivenNullType() {
+        DatafeedConfig.Builder conf = constructBuilder();
+        expectThrows(NullPointerException.class, () -> conf.setTypes(null));
+    }
+
+    public void testCheckValid_GivenNullQuery() {
+        DatafeedConfig.Builder conf = constructBuilder();
+        expectThrows(NullPointerException.class, () -> conf.setQuery(null));
+    }
+
+    public static String randomValidDatafeedId() {
+        CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray());
+        return generator.ofCodePointsLength(random(), 10, 10);
+    }
+
+    private static DatafeedConfig.Builder constructBuilder() {
+        return new DatafeedConfig.Builder(randomValidDatafeedId(), randomAlphaOfLength(10));
+    }
+
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedUpdateTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedUpdateTests.java
new file mode 100644
index 0000000000000..edbef8461e053
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedUpdateTests.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.datafeed;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class DatafeedUpdateTests extends AbstractXContentTestCase<DatafeedUpdate> {
+
+    @Override
+    protected DatafeedUpdate createTestInstance() {
+        DatafeedUpdate.Builder builder = new DatafeedUpdate.Builder(DatafeedConfigTests.randomValidDatafeedId());
+        if (randomBoolean()) {
+            builder.setJobId(randomAlphaOfLength(10));
+        }
+        if (randomBoolean()) {
+            builder.setQueryDelay(TimeValue.timeValueMillis(randomIntBetween(1, Integer.MAX_VALUE)));
+        }
+        if (randomBoolean()) {
+            builder.setFrequency(TimeValue.timeValueSeconds(randomIntBetween(1, Integer.MAX_VALUE)));
+        }
+        if (randomBoolean()) {
+            builder.setIndices(DatafeedConfigTests.randomStringList(1, 10));
+        }
+        if (randomBoolean()) {
+            builder.setTypes(DatafeedConfigTests.randomStringList(1, 10));
+        }
+        if (randomBoolean()) {
+            builder.setQuery(QueryBuilders.termQuery(randomAlphaOfLength(10), randomAlphaOfLength(10)));
+        }
+        if (randomBoolean()) {
+            int scriptsSize = randomInt(3);
+            List<SearchSourceBuilder.ScriptField> scriptFields = new ArrayList<>(scriptsSize);
+            for (int scriptIndex = 0; scriptIndex < scriptsSize; scriptIndex++) {
+                scriptFields.add(new SearchSourceBuilder.ScriptField(randomAlphaOfLength(10), mockScript(randomAlphaOfLength(10)),
+                    randomBoolean()));
+            }
+            builder.setScriptFields(scriptFields);
+        }
+        if (randomBoolean()) {
+            // can only test with a single agg as the xcontent order gets randomized by test base class and then
+            // the actual xcontent isn't the same and test fail.
+            // Testing with a single agg is ok as we don't have special list xcontent logic
+            AggregatorFactories.Builder aggs = new AggregatorFactories.Builder();
+            aggs.addAggregator(AggregationBuilders.avg(randomAlphaOfLength(10)).field(randomAlphaOfLength(10)));
+            builder.setAggregations(aggs);
+        }
+        if (randomBoolean()) {
+            builder.setScrollSize(randomIntBetween(0, Integer.MAX_VALUE));
+        }
+        if (randomBoolean()) {
+            builder.setChunkingConfig(ChunkingConfigTests.createRandomizedChunk());
+        }
+        return builder.build();
+    }
+
+    @Override
+    protected DatafeedUpdate doParseInstance(XContentParser parser) {
+        return DatafeedUpdate.PARSER.apply(parser, null).build();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        return new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfigTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfigTests.java
new file mode 100644
index 0000000000000..34f12fc067e75
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfigTests.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class AnalysisConfigTests extends AbstractXContentTestCase<AnalysisConfig> {
+
+    public static AnalysisConfig.Builder createRandomized() {
+        boolean isCategorization = randomBoolean();
+        List<Detector> detectors = new ArrayList<>();
+        int numDetectors = randomIntBetween(1, 10);
+        for (int i = 0; i < numDetectors; i++) {
+            Detector.Builder builder = new Detector.Builder("count", null);
+            builder.setPartitionFieldName(isCategorization ? "mlcategory" : "part");
+            detectors.add(builder.build());
+        }
+        AnalysisConfig.Builder builder = new AnalysisConfig.Builder(detectors);
+        if (randomBoolean()) {
+            TimeValue bucketSpan = TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000));
+            builder.setBucketSpan(bucketSpan);
+        }
+        if (isCategorization) {
+            builder.setCategorizationFieldName(randomAlphaOfLength(10));
+            if (randomBoolean()) {
+                builder.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false)));
+            } else {
+                CategorizationAnalyzerConfig.Builder analyzerBuilder = new CategorizationAnalyzerConfig.Builder();
+                if (rarely()) {
+                    analyzerBuilder.setAnalyzer(randomAlphaOfLength(10));
+                } else {
+                    if (randomBoolean()) {
+                        for (String pattern : generateRandomStringArray(3, 40, false)) {
+                            Map<String, Object> charFilter = new HashMap<>();
+                            charFilter.put("type", "pattern_replace");
+                            charFilter.put("pattern", pattern);
+                            analyzerBuilder.addCharFilter(charFilter);
+                        }
+                    }
+
+                    Map<String, Object> tokenizer = new HashMap<>();
+                    tokenizer.put("type", "pattern");
+                    tokenizer.put("pattern", randomAlphaOfLength(10));
+                    analyzerBuilder.setTokenizer(tokenizer);
+
+                    if (randomBoolean()) {
+                        for (String pattern : generateRandomStringArray(4, 40, false)) {
+                            Map<String, Object> tokenFilter = new HashMap<>();
+                            tokenFilter.put("type", "pattern_replace");
+                            tokenFilter.put("pattern", pattern);
+                            analyzerBuilder.addTokenFilter(tokenFilter);
+                        }
+                    }
+                }
+                builder.setCategorizationAnalyzerConfig(analyzerBuilder.build());
+            }
+        }
+        if (randomBoolean()) {
+            builder.setLatency(TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000)));
+        }
+        if (randomBoolean()) {
+            builder.setMultivariateByFields(randomBoolean());
+        }
+        if (randomBoolean()) {
+            builder.setOverlappingBuckets(randomBoolean());
+        }
+        if (randomBoolean()) {
+            builder.setResultFinalizationWindow(randomNonNegativeLong());
+        }
+
+        builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false)));
+        return builder;
+    }
+
+    @Override
+    protected AnalysisConfig createTestInstance() {
+        return createRandomized().build();
+    }
+
+    @Override
+    protected AnalysisConfig doParseInstance(XContentParser parser) {
+        return AnalysisConfig.PARSER.apply(parser, null).build();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public void testBuilder_WithNullDetectors() {
+        AnalysisConfig.Builder builder = new AnalysisConfig.Builder(new ArrayList<>());
+        NullPointerException ex = expectThrows(NullPointerException.class, () ->  builder.setDetectors(null));
+        assertEquals("[detectors] must not be null", ex.getMessage());
+    }
+
+    public void testEquals_GivenSameReference() {
+        AnalysisConfig config = createRandomized().build();
+        assertTrue(config.equals(config));
+    }
+
+    public void testEquals_GivenDifferentClass() {
+        assertFalse(createRandomized().build().equals("a string"));
+    }
+
+    public void testEquals_GivenNull() {
+        assertFalse(createRandomized().build().equals(null));
+    }
+
+    public void testEquals_GivenEqualConfig() {
+        AnalysisConfig config1 = createValidCategorizationConfig().build();
+        AnalysisConfig config2 = createValidCategorizationConfig().build();
+
+        assertTrue(config1.equals(config2));
+        assertTrue(config2.equals(config1));
+        assertEquals(config1.hashCode(), config2.hashCode());
+    }
+
+    public void testEquals_GivenDifferentBucketSpan() {
+        AnalysisConfig.Builder builder = createConfigBuilder();
+        builder.setBucketSpan(TimeValue.timeValueSeconds(1800));
+        AnalysisConfig config1 = builder.build();
+
+        builder = createConfigBuilder();
+        builder.setBucketSpan(TimeValue.timeValueHours(1));
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenCategorizationField() {
+        AnalysisConfig.Builder builder = createValidCategorizationConfig();
+        builder.setCategorizationFieldName("foo");
+        AnalysisConfig config1 = builder.build();
+
+        builder = createValidCategorizationConfig();
+        builder.setCategorizationFieldName("bar");
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenDifferentDetector() {
+        AnalysisConfig config1 = createConfigWithDetectors(Collections.singletonList(new Detector.Builder("min", "low_count").build()));
+
+        AnalysisConfig config2 = createConfigWithDetectors(Collections.singletonList(new Detector.Builder("min", "high_count").build()));
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenDifferentInfluencers() {
+        AnalysisConfig.Builder builder = createConfigBuilder();
+        builder.setInfluencers(Collections.singletonList("foo"));
+        AnalysisConfig config1 = builder.build();
+
+        builder = createConfigBuilder();
+        builder.setInfluencers(Collections.singletonList("bar"));
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenDifferentLatency() {
+        AnalysisConfig.Builder builder = createConfigBuilder();
+        builder.setLatency(TimeValue.timeValueSeconds(1800));
+        AnalysisConfig config1 = builder.build();
+
+        builder = createConfigBuilder();
+        builder.setLatency(TimeValue.timeValueSeconds(1801));
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenSummaryCountField() {
+        AnalysisConfig.Builder builder = createConfigBuilder();
+        builder.setSummaryCountFieldName("foo");
+        AnalysisConfig config1 = builder.build();
+
+        builder = createConfigBuilder();
+        builder.setSummaryCountFieldName("bar");
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenMultivariateByField() {
+        AnalysisConfig.Builder builder = createConfigBuilder();
+        builder.setMultivariateByFields(true);
+        AnalysisConfig config1 = builder.build();
+
+        builder = createConfigBuilder();
+        builder.setMultivariateByFields(false);
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenDifferentCategorizationFilters() {
+        AnalysisConfig.Builder configBuilder1 = createValidCategorizationConfig();
+        AnalysisConfig.Builder configBuilder2 = createValidCategorizationConfig();
+        configBuilder1.setCategorizationFilters(Arrays.asList("foo", "bar"));
+        configBuilder2.setCategorizationFilters(Arrays.asList("foo", "foobar"));
+        AnalysisConfig config1 = configBuilder1.build();
+        AnalysisConfig config2 = configBuilder2.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    private static AnalysisConfig createConfigWithDetectors(List<Detector> detectors) {
+        return new AnalysisConfig.Builder(detectors).build();
+    }
+
+    private static AnalysisConfig.Builder createConfigBuilder() {
+        return new AnalysisConfig.Builder(Collections.singletonList(new Detector.Builder("min", "count").build()));
+    }
+
+    private static AnalysisConfig.Builder createValidCategorizationConfig() {
+        Detector.Builder detector = new Detector.Builder("count", null);
+        detector.setByFieldName("mlcategory");
+        AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
+        analysisConfig.setBucketSpan(TimeValue.timeValueHours(1));
+        analysisConfig.setLatency(TimeValue.ZERO);
+        analysisConfig.setCategorizationFieldName("msg");
+        return analysisConfig;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        return new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisLimitsTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisLimitsTests.java
new file mode 100644
index 0000000000000..5003da10780d4
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisLimitsTests.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AnalysisLimitsTests extends AbstractXContentTestCase<AnalysisLimits> {
+
+    @Override
+    protected AnalysisLimits createTestInstance() {
+        return createRandomized();
+    }
+
+    public static AnalysisLimits createRandomized() {
+        return new AnalysisLimits(randomBoolean() ? (long) randomIntBetween(1, 1000000) : null,
+                randomBoolean() ? randomNonNegativeLong() : null);
+    }
+
+    @Override
+    protected AnalysisLimits doParseInstance(XContentParser parser) {
+        return AnalysisLimits.PARSER.apply(parser, null);
+    }
+
+    public void testParseModelMemoryLimitGivenPositiveNumber() throws IOException {
+        String json = "{\"model_memory_limit\": 2048}";
+        XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+                .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);
+
+        AnalysisLimits limits = AnalysisLimits.PARSER.apply(parser, null);
+
+        assertThat(limits.getModelMemoryLimit(), equalTo(2048L));
+    }
+
+    public void testParseModelMemoryLimitGivenStringMultipleOfMBs() throws IOException {
+        String json = "{\"model_memory_limit\":\"4g\"}";
+        XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+                .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);
+
+        AnalysisLimits limits = AnalysisLimits.PARSER.apply(parser, null);
+
+        assertThat(limits.getModelMemoryLimit(), equalTo(4096L));
+    }
+
+    public void testEquals_GivenEqual() {
+        AnalysisLimits analysisLimits1 = new AnalysisLimits(10L, 20L);
+        AnalysisLimits analysisLimits2 = new AnalysisLimits(10L, 20L);
+
+        assertTrue(analysisLimits1.equals(analysisLimits1));
+        assertTrue(analysisLimits1.equals(analysisLimits2));
+        assertTrue(analysisLimits2.equals(analysisLimits1));
+    }
+
+    public void testEquals_GivenDifferentModelMemoryLimit() {
+        AnalysisLimits analysisLimits1 = new AnalysisLimits(10L, 20L);
+        AnalysisLimits analysisLimits2 = new AnalysisLimits(11L, 20L);
+
+        assertFalse(analysisLimits1.equals(analysisLimits2));
+        assertFalse(analysisLimits2.equals(analysisLimits1));
+    }
+
+    public void testEquals_GivenDifferentCategorizationExamplesLimit() {
+        AnalysisLimits analysisLimits1 = new AnalysisLimits(10L, 20L);
+        AnalysisLimits analysisLimits2 = new AnalysisLimits(10L, 21L);
+
+        assertFalse(analysisLimits1.equals(analysisLimits2));
+        assertFalse(analysisLimits2.equals(analysisLimits1));
+    }
+
+    public void testHashCode_GivenEqual() {
+        AnalysisLimits analysisLimits1 = new AnalysisLimits(5555L, 3L);
+        AnalysisLimits analysisLimits2 = new AnalysisLimits(5555L, 3L);
+
+        assertEquals(analysisLimits1.hashCode(), analysisLimits2.hashCode());
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/CategorizationAnalyzerConfigTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/CategorizationAnalyzerConfigTests.java
new file mode 100644
index 0000000000000..36fb51ed10e72
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/CategorizationAnalyzerConfigTests.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class CategorizationAnalyzerConfigTests extends AbstractXContentTestCase<CategorizationAnalyzerConfig> {
+
+    @Override
+    protected CategorizationAnalyzerConfig createTestInstance() {
+        return createRandomized().build();
+    }
+
+    public static CategorizationAnalyzerConfig.Builder createRandomized() {
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder();
+        if (rarely()) {
+            builder.setAnalyzer(randomAlphaOfLength(10));
+        } else {
+            if (randomBoolean()) {
+                for (String pattern : generateRandomStringArray(3, 40, false)) {
+                    if (rarely()) {
+                        builder.addCharFilter(randomAlphaOfLength(10));
+                    } else {
+                        Map<String, Object> charFilter = new HashMap<>();
+                        charFilter.put("type", "pattern_replace");
+                        charFilter.put("pattern", pattern);
+                        builder.addCharFilter(charFilter);
+                    }
+                }
+            }
+
+            if (rarely()) {
+                builder.setTokenizer(randomAlphaOfLength(10));
+            } else {
+                Map<String, Object> tokenizer = new HashMap<>();
+                tokenizer.put("type", "pattern");
+                tokenizer.put("pattern", randomAlphaOfLength(10));
+                builder.setTokenizer(tokenizer);
+            }
+
+            if (randomBoolean()) {
+                for (String pattern : generateRandomStringArray(4, 40, false)) {
+                    if (rarely()) {
+                        builder.addTokenFilter(randomAlphaOfLength(10));
+                    } else {
+                        Map<String, Object> tokenFilter = new HashMap<>();
+                        tokenFilter.put("type", "pattern_replace");
+                        tokenFilter.put("pattern", pattern);
+                        builder.addTokenFilter(tokenFilter);
+                    }
+                }
+            }
+        }
+        return builder;
+    }
+
+    @Override
+    protected CategorizationAnalyzerConfig doParseInstance(XContentParser parser) throws IOException {
+        return CategorizationAnalyzerConfig.buildFromXContentObject(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DataDescriptionTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DataDescriptionTests.java
new file mode 100644
index 0000000000000..8ca2dc494f3c5
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DataDescriptionTests.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.protocol.xpack.ml.job.config.DataDescription.DataFormat;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.core.Is.is;
+
+public class DataDescriptionTests extends AbstractXContentTestCase<DataDescription> {
+
+    public void testDefault() {
+        DataDescription dataDescription = new DataDescription.Builder().build();
+        assertThat(dataDescription.getFormat(), equalTo(DataFormat.XCONTENT));
+        assertThat(dataDescription.getTimeField(), equalTo("time"));
+        assertThat(dataDescription.getTimeFormat(), equalTo("epoch_ms"));
+        assertThat(dataDescription.getFieldDelimiter(), is(nullValue()));
+        assertThat(dataDescription.getQuoteCharacter(), is(nullValue()));
+    }
+
+    public void testDefaultDelimited() {
+        DataDescription.Builder dataDescriptionBuilder = new DataDescription.Builder();
+        dataDescriptionBuilder.setFormat(DataFormat.DELIMITED);
+        DataDescription dataDescription = dataDescriptionBuilder.build();
+
+        assertThat(dataDescription.getFormat(), equalTo(DataFormat.DELIMITED));
+        assertThat(dataDescription.getTimeField(), equalTo("time"));
+        assertThat(dataDescription.getTimeFormat(), equalTo("epoch_ms"));
+        assertThat(dataDescription.getFieldDelimiter(), is('\t'));
+        assertThat(dataDescription.getQuoteCharacter(), is('"'));
+    }
+
+    public void testEquals_GivenDifferentDateFormat() {
+        DataDescription.Builder description1 = new DataDescription.Builder();
+        description1.setFormat(DataFormat.XCONTENT);
+        description1.setQuoteCharacter('"');
+        description1.setTimeField("timestamp");
+        description1.setTimeFormat("epoch");
+        description1.setFieldDelimiter(',');
+
+        DataDescription.Builder description2 = new DataDescription.Builder();
+        description2.setFormat(DataFormat.DELIMITED);
+        description2.setQuoteCharacter('"');
+        description2.setTimeField("timestamp");
+        description2.setTimeFormat("epoch");
+        description2.setFieldDelimiter(',');
+
+        assertFalse(description1.build().equals(description2.build()));
+        assertFalse(description2.build().equals(description1.build()));
+    }
+
+    public void testEquals_GivenDifferentQuoteCharacter() {
+        DataDescription.Builder description1 = new DataDescription.Builder();
+        description1.setFormat(DataFormat.XCONTENT);
+        description1.setQuoteCharacter('"');
+        description1.setTimeField("timestamp");
+        description1.setTimeFormat("epoch");
+        description1.setFieldDelimiter(',');
+
+        DataDescription.Builder description2 = new DataDescription.Builder();
+        description2.setFormat(DataFormat.XCONTENT);
+        description2.setQuoteCharacter('\'');
+        description2.setTimeField("timestamp");
+        description2.setTimeFormat("epoch");
+        description2.setFieldDelimiter(',');
+
+        assertFalse(description1.build().equals(description2.build()));
+        assertFalse(description2.build().equals(description1.build()));
+    }
+
+    public void testEquals_GivenDifferentTimeField() {
+        DataDescription.Builder description1 = new DataDescription.Builder();
+        description1.setFormat(DataFormat.XCONTENT);
+        description1.setQuoteCharacter('"');
+        description1.setTimeField("timestamp");
+        description1.setTimeFormat("epoch");
+        description1.setFieldDelimiter(',');
+
+        DataDescription.Builder description2 = new DataDescription.Builder();
+        description2.setFormat(DataFormat.XCONTENT);
+        description2.setQuoteCharacter('"');
+        description2.setTimeField("time");
+        description2.setTimeFormat("epoch");
+        description2.setFieldDelimiter(',');
+
+        assertFalse(description1.build().equals(description2.build()));
+        assertFalse(description2.build().equals(description1.build()));
+    }
+
+    public void testEquals_GivenDifferentTimeFormat() {
+        DataDescription.Builder description1 = new DataDescription.Builder();
+        description1.setFormat(DataFormat.XCONTENT);
+        description1.setQuoteCharacter('"');
+        description1.setTimeField("timestamp");
+        description1.setTimeFormat("epoch");
+        description1.setFieldDelimiter(',');
+
+        DataDescription.Builder description2 = new DataDescription.Builder();
+        description2.setFormat(DataFormat.XCONTENT);
+        description2.setQuoteCharacter('"');
+        description2.setTimeField("timestamp");
+        description2.setTimeFormat("epoch_ms");
+        description2.setFieldDelimiter(',');
+
+        assertFalse(description1.build().equals(description2.build()));
+        assertFalse(description2.build().equals(description1.build()));
+    }
+
+    public void testEquals_GivenDifferentFieldDelimiter() {
+        DataDescription.Builder description1 = new DataDescription.Builder();
+        description1.setFormat(DataFormat.XCONTENT);
+        description1.setQuoteCharacter('"');
+        description1.setTimeField("timestamp");
+        description1.setTimeFormat("epoch");
+        description1.setFieldDelimiter(',');
+
+        DataDescription.Builder description2 = new DataDescription.Builder();
+        description2.setFormat(DataFormat.XCONTENT);
+        description2.setQuoteCharacter('"');
+        description2.setTimeField("timestamp");
+        description2.setTimeFormat("epoch");
+        description2.setFieldDelimiter(';');
+
+        assertFalse(description1.build().equals(description2.build()));
+        assertFalse(description2.build().equals(description1.build()));
+    }
+
+    @Override
+    protected DataDescription createTestInstance() {
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        if (randomBoolean()) {
+            dataDescription.setFormat(randomFrom(DataFormat.values()));
+        }
+        if (randomBoolean()) {
+            dataDescription.setTimeField(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            String format;
+            if (randomBoolean()) {
+                format = DataDescription.EPOCH;
+            } else if (randomBoolean()) {
+                format = DataDescription.EPOCH_MS;
+            } else {
+                format = "yyyy-MM-dd HH:mm:ss.SSS";
+            }
+            dataDescription.setTimeFormat(format);
+        }
+        if (randomBoolean()) {
+            dataDescription.setFieldDelimiter(randomAlphaOfLength(1).charAt(0));
+        }
+        if (randomBoolean()) {
+            dataDescription.setQuoteCharacter(randomAlphaOfLength(1).charAt(0));
+        }
+        return dataDescription.build();
+    }
+
+    @Override
+    protected DataDescription doParseInstance(XContentParser parser) {
+        return DataDescription.PARSER.apply(parser, null).build();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRuleTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRuleTests.java
new file mode 100644
index 0000000000000..bc70a404894ab
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRuleTests.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class DetectionRuleTests extends AbstractXContentTestCase<DetectionRule> {
+
+    @Override
+    protected DetectionRule createTestInstance() {
+        DetectionRule.Builder builder = new DetectionRule.Builder();
+
+        if (randomBoolean()) {
+            EnumSet<RuleAction> actions = EnumSet.noneOf(RuleAction.class);
+            int actionsCount = randomIntBetween(1, RuleAction.values().length);
+            for (int i = 0; i < actionsCount; ++i) {
+                actions.add(randomFrom(RuleAction.values()));
+            }
+            builder.setActions(actions);
+        }
+
+        boolean hasScope = randomBoolean();
+        boolean hasConditions = randomBoolean();
+
+        if (!hasScope && !hasConditions) {
+            // at least one of the two should be present
+            if (randomBoolean()) {
+                hasScope = true;
+            } else {
+                hasConditions = true;
+            }
+        }
+
+        if (hasScope) {
+            Map<String, FilterRef> scope = new HashMap<>();
+            int scopeSize = randomIntBetween(1, 3);
+            for (int i = 0; i < scopeSize; i++) {
+                scope.put(randomAlphaOfLength(20), new FilterRef(randomAlphaOfLength(20), randomFrom(FilterRef.FilterType.values())));
+            }
+            builder.setScope(new RuleScope(scope));
+        }
+
+        if (hasConditions) {
+            int size = randomIntBetween(1, 5);
+            List<RuleCondition> ruleConditions = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                // no need for random condition (it is already tested)
+                ruleConditions.addAll(createCondition(randomDouble()));
+            }
+            builder.setConditions(ruleConditions);
+        }
+
+        return builder.build();
+    }
+
+    @Override
+    protected DetectionRule doParseInstance(XContentParser parser) {
+        return DetectionRule.PARSER.apply(parser, null).build();
+    }
+
+    private static List<RuleCondition> createCondition(double value) {
+        return Collections.singletonList(new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, value));
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorTests.java
new file mode 100644
index 0000000000000..0b1ba892acd30
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorTests.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+
+public class DetectorTests extends AbstractXContentTestCase<Detector> {
+
+    public void testEquals_GivenEqual() {
+        Detector.Builder builder = new Detector.Builder("mean", "field");
+        builder.setByFieldName("by_field");
+        builder.setOverFieldName("over_field");
+        builder.setPartitionFieldName("partition");
+        builder.setUseNull(false);
+        Detector detector1 = builder.build();
+
+        builder = new Detector.Builder("mean", "field");
+        builder.setByFieldName("by_field");
+        builder.setOverFieldName("over_field");
+        builder.setPartitionFieldName("partition");
+        builder.setUseNull(false);
+        Detector detector2 = builder.build();
+
+        assertTrue(detector1.equals(detector2));
+        assertTrue(detector2.equals(detector1));
+        assertEquals(detector1.hashCode(), detector2.hashCode());
+    }
+
+    public void testEquals_GivenDifferentDetectorDescription() {
+        Detector detector1 = createDetector().build();
+        Detector.Builder builder = createDetector();
+        builder.setDetectorDescription("bar");
+        Detector detector2 = builder.build();
+
+        assertFalse(detector1.equals(detector2));
+    }
+
+    public void testEquals_GivenDifferentByFieldName() {
+        Detector detector1 = createDetector().build();
+        Detector detector2 = createDetector().build();
+
+        assertEquals(detector1, detector2);
+
+        Detector.Builder builder = new Detector.Builder(detector2);
+        builder.setByFieldName("by2");
+        detector2 = builder.build();
+        assertFalse(detector1.equals(detector2));
+    }
+
+    private Detector.Builder createDetector() {
+        Detector.Builder detector = new Detector.Builder("mean", "field");
+        detector.setByFieldName("by_field");
+        detector.setOverFieldName("over_field");
+        detector.setPartitionFieldName("partition");
+        detector.setUseNull(true);
+        DetectionRule rule = new DetectionRule.Builder(RuleScope.builder().exclude("partition", "partition_filter"))
+                .setActions(RuleAction.SKIP_RESULT)
+                .build();
+        detector.setRules(Collections.singletonList(rule));
+        return detector;
+    }
+
+    @Override
+    protected Detector createTestInstance() {
+        DetectorFunction function = randomFrom(EnumSet.allOf(DetectorFunction.class));
+        Detector.Builder detector = new Detector.Builder(function, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 20));
+        if (randomBoolean()) {
+            detector.setDetectorDescription(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            detector.setPartitionFieldName(randomAlphaOfLengthBetween(6, 20));
+        } else if (randomBoolean()) {
+            detector.setOverFieldName(randomAlphaOfLengthBetween(6, 20));
+        } else if (randomBoolean()) {
+            detector.setByFieldName(randomAlphaOfLengthBetween(6, 20));
+        }
+        if (randomBoolean()) {
+            detector.setExcludeFrequent(randomFrom(Detector.ExcludeFrequent.values()));
+        }
+        if (randomBoolean()) {
+            int size = randomInt(10);
+            List<DetectionRule> rules = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                // no need for random DetectionRule (it is already tested)
+                rules.add(new DetectionRule.Builder(Collections.singletonList(RuleConditionTests.createRandom())).build());
+            }
+            detector.setRules(rules);
+        }
+        if (randomBoolean()) {
+            detector.setUseNull(randomBoolean());
+        }
+        return detector.build();
+    }
+
+    @Override
+    protected Detector doParseInstance(XContentParser parser) {
+        return Detector.PARSER.apply(parser, null).build();
+    }
+
+    public void testExcludeFrequentForString() {
+        assertEquals(Detector.ExcludeFrequent.ALL, Detector.ExcludeFrequent.forString("all"));
+        assertEquals(Detector.ExcludeFrequent.ALL, Detector.ExcludeFrequent.forString("ALL"));
+        assertEquals(Detector.ExcludeFrequent.NONE, Detector.ExcludeFrequent.forString("none"));
+        assertEquals(Detector.ExcludeFrequent.NONE, Detector.ExcludeFrequent.forString("NONE"));
+        assertEquals(Detector.ExcludeFrequent.BY, Detector.ExcludeFrequent.forString("by"));
+        assertEquals(Detector.ExcludeFrequent.BY, Detector.ExcludeFrequent.forString("BY"));
+        assertEquals(Detector.ExcludeFrequent.OVER, Detector.ExcludeFrequent.forString("over"));
+        assertEquals(Detector.ExcludeFrequent.OVER, Detector.ExcludeFrequent.forString("OVER"));
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRefTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRefTests.java
new file mode 100644
index 0000000000000..00862e5307b9e
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRefTests.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class FilterRefTests extends AbstractXContentTestCase<FilterRef> {
+
+    @Override
+    protected FilterRef createTestInstance() {
+        return new FilterRef(randomAlphaOfLength(20), randomFrom(FilterRef.FilterType.values()));
+    }
+
+    @Override
+    protected FilterRef doParseInstance(XContentParser parser) throws IOException {
+        return FilterRef.PARSER.parse(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/JobTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/JobTests.java
new file mode 100644
index 0000000000000..7ba4946efa753
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/JobTests.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class JobTests extends AbstractXContentTestCase<Job> {
+
+    private static final String FUTURE_JOB = "{\n" +
+        "    \"job_id\": \"farequote\",\n" +
+        "    \"create_time\": 1234567890000,\n" +
+        "    \"tomorrows_technology_today\": \"wow\",\n" +
+        "    \"analysis_config\": {\n" +
+        "        \"bucket_span\": \"1h\",\n" +
+        "        \"something_new\": \"gasp\",\n" +
+        "        \"detectors\": [{\"function\": \"metric\", \"field_name\": \"responsetime\", \"by_field_name\": \"airline\"}]\n" +
+        "    },\n" +
+        "    \"data_description\": {\n" +
+        "        \"time_field\": \"time\",\n" +
+        "        \"the_future\": 123\n" +
+        "    }\n" +
+        "}";
+
+    @Override
+    protected Job createTestInstance() {
+        return createRandomizedJob();
+    }
+
+    @Override
+    protected Job doParseInstance(XContentParser parser) {
+        return Job.PARSER.apply(parser, null).build();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public void testFutureMetadataParse() throws IOException {
+        XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, FUTURE_JOB);
+        // The parser should tolerate unknown fields
+        assertNotNull(Job.PARSER.apply(parser, null).build());
+    }
+
+    public void testEquals_GivenDifferentClass() {
+        Job job = buildJobBuilder("foo").build();
+        assertFalse(job.equals("a string"));
+    }
+
+    public void testEquals_GivenDifferentIds() {
+        Date createTime = new Date();
+        Job.Builder builder = buildJobBuilder("foo");
+        builder.setCreateTime(createTime);
+        Job job1 = builder.build();
+        builder.setId("bar");
+        Job job2 = builder.build();
+        assertFalse(job1.equals(job2));
+    }
+
+    public void testEquals_GivenDifferentRenormalizationWindowDays() {
+        Date date = new Date();
+        Job.Builder jobDetails1 = new Job.Builder("foo");
+        jobDetails1.setDataDescription(new DataDescription.Builder());
+        jobDetails1.setAnalysisConfig(createAnalysisConfig());
+        jobDetails1.setRenormalizationWindowDays(3L);
+        jobDetails1.setCreateTime(date);
+        Job.Builder jobDetails2 = new Job.Builder("foo");
+        jobDetails2.setDataDescription(new DataDescription.Builder());
+        jobDetails2.setRenormalizationWindowDays(4L);
+        jobDetails2.setAnalysisConfig(createAnalysisConfig());
+        jobDetails2.setCreateTime(date);
+        assertFalse(jobDetails1.build().equals(jobDetails2.build()));
+    }
+
+    public void testEquals_GivenDifferentBackgroundPersistInterval() {
+        Date date = new Date();
+        Job.Builder jobDetails1 = new Job.Builder("foo");
+        jobDetails1.setDataDescription(new DataDescription.Builder());
+        jobDetails1.setAnalysisConfig(createAnalysisConfig());
+        jobDetails1.setBackgroundPersistInterval(TimeValue.timeValueSeconds(10000L));
+        jobDetails1.setCreateTime(date);
+        Job.Builder jobDetails2 = new Job.Builder("foo");
+        jobDetails2.setDataDescription(new DataDescription.Builder());
+        jobDetails2.setBackgroundPersistInterval(TimeValue.timeValueSeconds(8000L));
+        jobDetails2.setAnalysisConfig(createAnalysisConfig());
+        jobDetails2.setCreateTime(date);
+        assertFalse(jobDetails1.build().equals(jobDetails2.build()));
+    }
+
+    public void testEquals_GivenDifferentModelSnapshotRetentionDays() {
+        Date date = new Date();
+        Job.Builder jobDetails1 = new Job.Builder("foo");
+        jobDetails1.setDataDescription(new DataDescription.Builder());
+        jobDetails1.setAnalysisConfig(createAnalysisConfig());
+        jobDetails1.setModelSnapshotRetentionDays(10L);
+        jobDetails1.setCreateTime(date);
+        Job.Builder jobDetails2 = new Job.Builder("foo");
+        jobDetails2.setDataDescription(new DataDescription.Builder());
+        jobDetails2.setModelSnapshotRetentionDays(8L);
+        jobDetails2.setAnalysisConfig(createAnalysisConfig());
+        jobDetails2.setCreateTime(date);
+        assertFalse(jobDetails1.build().equals(jobDetails2.build()));
+    }
+
+    public void testEquals_GivenDifferentResultsRetentionDays() {
+        Date date = new Date();
+        Job.Builder jobDetails1 = new Job.Builder("foo");
+        jobDetails1.setDataDescription(new DataDescription.Builder());
+        jobDetails1.setAnalysisConfig(createAnalysisConfig());
+        jobDetails1.setCreateTime(date);
+        jobDetails1.setResultsRetentionDays(30L);
+        Job.Builder jobDetails2 = new Job.Builder("foo");
+        jobDetails2.setDataDescription(new DataDescription.Builder());
+        jobDetails2.setResultsRetentionDays(4L);
+        jobDetails2.setAnalysisConfig(createAnalysisConfig());
+        jobDetails2.setCreateTime(date);
+        assertFalse(jobDetails1.build().equals(jobDetails2.build()));
+    }
+
+    public void testEquals_GivenDifferentCustomSettings() {
+        Job.Builder jobDetails1 = buildJobBuilder("foo");
+        Map<String, Object> customSettings1 = new HashMap<>();
+        customSettings1.put("key1", "value1");
+        jobDetails1.setCustomSettings(customSettings1);
+        Job.Builder jobDetails2 = buildJobBuilder("foo");
+        Map<String, Object> customSettings2 = new HashMap<>();
+        customSettings2.put("key2", "value2");
+        jobDetails2.setCustomSettings(customSettings2);
+        assertFalse(jobDetails1.build().equals(jobDetails2.build()));
+    }
+
+    public void testCopyConstructor() {
+        for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
+            Job job = createTestInstance();
+            Job copy = new Job.Builder(job).build();
+            assertEquals(job, copy);
+        }
+    }
+
+    public void testBuilder_WithNullID() {
+        Job.Builder builder = new Job.Builder("anything").setId(null);
+        NullPointerException ex = expectThrows(NullPointerException.class, builder::build);
+        assertEquals("[job_id] must not be null", ex.getMessage());
+    }
+
+    public void testBuilder_WithNullJobType() {
+        Job.Builder builder = new Job.Builder("anything").setJobType(null);
+        NullPointerException ex = expectThrows(NullPointerException.class, builder::build);
+        assertEquals("[job_type] must not be null", ex.getMessage());
+    }
+
+    public static Job.Builder buildJobBuilder(String id, Date date) {
+        Job.Builder builder = new Job.Builder(id);
+        builder.setCreateTime(date);
+        AnalysisConfig.Builder ac = createAnalysisConfig();
+        DataDescription.Builder dc = new DataDescription.Builder();
+        builder.setAnalysisConfig(ac);
+        builder.setDataDescription(dc);
+        return builder;
+    }
+
+    public static Job.Builder buildJobBuilder(String id) {
+        return buildJobBuilder(id, new Date());
+    }
+
+    public static String randomValidJobId() {
+        CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray());
+        return generator.ofCodePointsLength(random(), 10, 10);
+    }
+
+    public static AnalysisConfig.Builder createAnalysisConfig() {
+        Detector.Builder d1 = new Detector.Builder("info_content", "domain");
+        d1.setOverFieldName("client");
+        Detector.Builder d2 = new Detector.Builder("min", "field");
+        return new AnalysisConfig.Builder(Arrays.asList(d1.build(), d2.build()));
+    }
+
+    public static Job createRandomizedJob() {
+        String jobId = randomValidJobId();
+        Job.Builder builder = new Job.Builder(jobId);
+        if (randomBoolean()) {
+            builder.setDescription(randomAlphaOfLength(10));
+        }
+        if (randomBoolean()) {
+            int groupsNum = randomIntBetween(0, 10);
+            List<String> groups = new ArrayList<>(groupsNum);
+            for (int i = 0; i < groupsNum; i++) {
+                groups.add(randomValidJobId());
+            }
+            builder.setGroups(groups);
+        }
+        builder.setCreateTime(new Date(randomNonNegativeLong()));
+        if (randomBoolean()) {
+            builder.setFinishedTime(new Date(randomNonNegativeLong()));
+        }
+        if (randomBoolean()) {
+            builder.setLastDataTime(new Date(randomNonNegativeLong()));
+        }
+        if (randomBoolean()) {
+            builder.setEstablishedModelMemory(randomNonNegativeLong());
+        }
+        builder.setAnalysisConfig(AnalysisConfigTests.createRandomized());
+        builder.setAnalysisLimits(AnalysisLimitsTests.createRandomized());
+
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setFormat(randomFrom(DataDescription.DataFormat.values()));
+        builder.setDataDescription(dataDescription);
+
+        if (randomBoolean()) {
+            builder.setModelPlotConfig(new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10)));
+        }
+        if (randomBoolean()) {
+            builder.setRenormalizationWindowDays(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            builder.setBackgroundPersistInterval(TimeValue.timeValueHours(randomIntBetween(1, 24)));
+        }
+        if (randomBoolean()) {
+            builder.setModelSnapshotRetentionDays(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            builder.setResultsRetentionDays(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            builder.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10)));
+        }
+        if (randomBoolean()) {
+            builder.setModelSnapshotId(randomAlphaOfLength(10));
+        }
+        if (randomBoolean()) {
+            builder.setResultsIndexName(randomValidJobId());
+        }
+        return builder.build();
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        return new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilterTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilterTests.java
new file mode 100644
index 0000000000000..6c595e2d6da15
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilterTests.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import static org.hamcrest.Matchers.contains;
+
+public class MlFilterTests extends AbstractXContentTestCase<MlFilter> {
+
+    public static MlFilter createTestFilter() {
+        return new MlFilterTests().createTestInstance();
+    }
+
+    @Override
+    protected MlFilter createTestInstance() {
+        return createRandom();
+    }
+
+    public static MlFilter createRandom() {
+        return createRandom(randomAlphaOfLength(10));
+    }
+
+    public static MlFilter createRandom(String filterId) {
+        String description = null;
+        if (randomBoolean()) {
+            description = randomAlphaOfLength(20);
+        }
+
+        int size = randomInt(10);
+        SortedSet<String> items = new TreeSet<>();
+        for (int i = 0; i < size; i++) {
+            items.add(randomAlphaOfLengthBetween(1, 20));
+        }
+        return MlFilter.builder(filterId).setDescription(description).setItems(items).build();
+    }
+
+    @Override
+    protected MlFilter doParseInstance(XContentParser parser) {
+        return MlFilter.PARSER.apply(parser, null).build();
+    }
+
+    public void testNullId() {
+        expectThrows(NullPointerException.class, () -> MlFilter.builder(null).build());
+    }
+
+    public void testNullItems() {
+        expectThrows(NullPointerException.class,
+            () -> MlFilter.builder(randomAlphaOfLength(10)).setItems((SortedSet<String>) null).build());
+    }
+
+    public void testItemsAreSorted() {
+        MlFilter filter = MlFilter.builder("foo").setItems("c", "b", "a").build();
+        assertThat(filter.getItems(), contains("a", "b", "c"));
+    }
+
+    public void testGetItemsReturnsUnmodifiable() {
+        MlFilter filter = MlFilter.builder("foo").setItems("c", "b", "a").build();
+        expectThrows(UnsupportedOperationException.class, () -> filter.getItems().add("x"));
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/ModelPlotConfigTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/ModelPlotConfigTests.java
new file mode 100644
index 0000000000000..23f13c732123a
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/ModelPlotConfigTests.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+public class ModelPlotConfigTests extends AbstractXContentTestCase<ModelPlotConfig> {
+
+    @Override
+    protected ModelPlotConfig createTestInstance() {
+        return new ModelPlotConfig(randomBoolean(), randomAlphaOfLengthBetween(1, 30));
+    }
+
+    @Override
+    protected ModelPlotConfig doParseInstance(XContentParser parser) {
+        return ModelPlotConfig.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleConditionTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleConditionTests.java
new file mode 100644
index 0000000000000..4348ea194d01b
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleConditionTests.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+public class RuleConditionTests extends AbstractXContentTestCase<RuleCondition> {
+
+    @Override
+    protected RuleCondition createTestInstance() {
+        return createRandom();
+    }
+
+    public static RuleCondition createRandom() {
+        RuleCondition.AppliesTo appliesTo = randomFrom(RuleCondition.AppliesTo.values());
+        Operator operator = randomFrom(Operator.LT, Operator.LTE, Operator.GT, Operator.GTE);
+        return new RuleCondition(appliesTo, operator, randomDouble());
+    }
+
+    @Override
+    protected RuleCondition doParseInstance(XContentParser parser) {
+        return RuleCondition.PARSER.apply(parser, null);
+    }
+
+    public void testEqualsGivenSameObject() {
+        RuleCondition condition = createRandom();
+        assertTrue(condition.equals(condition));
+    }
+
+    public void testEqualsGivenString() {
+        assertFalse(createRandom().equals("a string"));
+    }
+
+    public void testCreateTimeBased() {
+        RuleCondition timeBased = RuleCondition.createTime(Operator.GTE, 100L);
+        assertEquals(RuleCondition.AppliesTo.TIME, timeBased.getAppliesTo());
+        assertEquals(Operator.GTE, timeBased.getOperator());
+        assertEquals(100.0, timeBased.getValue(), 0.000001);
+    }
+
+    public void testAppliesToFromString() {
+        assertEquals(RuleCondition.AppliesTo.ACTUAL, RuleCondition.AppliesTo.fromString("actual"));
+        assertEquals(RuleCondition.AppliesTo.ACTUAL, RuleCondition.AppliesTo.fromString("ACTUAL"));
+        assertEquals(RuleCondition.AppliesTo.TYPICAL, RuleCondition.AppliesTo.fromString("typical"));
+        assertEquals(RuleCondition.AppliesTo.TYPICAL, RuleCondition.AppliesTo.fromString("TYPICAL"));
+        assertEquals(RuleCondition.AppliesTo.DIFF_FROM_TYPICAL, RuleCondition.AppliesTo.fromString("diff_from_typical"));
+        assertEquals(RuleCondition.AppliesTo.DIFF_FROM_TYPICAL, RuleCondition.AppliesTo.fromString("DIFF_FROM_TYPICAL"));
+        assertEquals(RuleCondition.AppliesTo.TIME, RuleCondition.AppliesTo.fromString("time"));
+        assertEquals(RuleCondition.AppliesTo.TIME, RuleCondition.AppliesTo.fromString("TIME"));
+    }
+
+    public void testAppliesToToString() {
+        assertEquals("actual", RuleCondition.AppliesTo.ACTUAL.toString());
+        assertEquals("typical", RuleCondition.AppliesTo.TYPICAL.toString());
+        assertEquals("diff_from_typical", RuleCondition.AppliesTo.DIFF_FROM_TYPICAL.toString());
+        assertEquals("time", RuleCondition.AppliesTo.TIME.toString());
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScopeTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScopeTests.java
new file mode 100644
index 0000000000000..ac97e457ac470
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScopeTests.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.contains;
+
+public class RuleScopeTests extends AbstractXContentTestCase<RuleScope> {
+
+    @Override
+    protected RuleScope createTestInstance() {
+        RuleScope.Builder scope = RuleScope.builder();
+        int count = randomIntBetween(0, 3);
+        for (int i = 0; i < count; ++i) {
+            if (randomBoolean()) {
+                scope.include(randomAlphaOfLength(20), randomAlphaOfLength(20));
+            } else {
+                scope.exclude(randomAlphaOfLength(20), randomAlphaOfLength(20));
+            }
+        }
+        return scope.build();
+    }
+
+    public void testGetReferencedFilters_GivenEmpty() {
+        assertTrue(RuleScope.builder().build().getReferencedFilters().isEmpty());
+    }
+
+    public void testGetReferencedFilters_GivenMultipleFields() {
+        RuleScope scope = RuleScope.builder()
+                .include("foo", "filter1")
+                .exclude("bar", "filter2")
+                .include("foobar", "filter3")
+                .build();
+        assertThat(scope.getReferencedFilters(), contains("filter1", "filter2", "filter3"));
+    }
+
+    @Override
+    protected RuleScope doParseInstance(XContentParser parser) throws IOException {
+        return RuleScope.parser().parse(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCountsTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCountsTests.java
new file mode 100644
index 0000000000000..2232e8c88d924
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCountsTests.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.process;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.joda.time.DateTime;
+
+import java.util.Date;
+
+import static org.hamcrest.Matchers.greaterThan;
+
+public class DataCountsTests extends AbstractXContentTestCase<DataCounts> {
+
+    public static DataCounts createTestInstance(String jobId) {
+        return new DataCounts(jobId, randomIntBetween(1, 1_000_000),
+                randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000),
+                randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000),
+                randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000),
+                new DateTime(randomDateTimeZone()).toDate(), new DateTime(randomDateTimeZone()).toDate(),
+                new DateTime(randomDateTimeZone()).toDate(), new DateTime(randomDateTimeZone()).toDate(),
+                new DateTime(randomDateTimeZone()).toDate());
+    }
+
+    @Override
+    public DataCounts createTestInstance() {
+        return createTestInstance(randomAlphaOfLength(10));
+    }
+
+    @Override
+    protected DataCounts doParseInstance(XContentParser parser) {
+        return DataCounts.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    public void testCountsEquals_GivenEqualCounts() {
+        DataCounts counts1 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
+        DataCounts counts2 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
+
+        assertTrue(counts1.equals(counts2));
+        assertTrue(counts2.equals(counts1));
+    }
+
+    public void testCountsHashCode_GivenEqualCounts() {
+        DataCounts counts1 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
+        DataCounts counts2 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
+        assertEquals(counts1.hashCode(), counts2.hashCode());
+    }
+
+    public void testCountsCopyConstructor() {
+        DataCounts counts1 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
+        DataCounts counts2 = new DataCounts(counts1);
+
+        assertEquals(counts1.hashCode(), counts2.hashCode());
+    }
+
+    public void testCountCreatedZero() throws Exception {
+        DataCounts counts = new DataCounts(randomAlphaOfLength(16));
+        assertAllFieldsEqualZero(counts);
+    }
+
+    public void testCountCopyCreatedFieldsNotZero() throws Exception {
+        DataCounts counts1 = createCounts(1, 200, 400, 3, 4, 5, 6, 7, 8, 9, 1479211200000L, 1479384000000L, 13, 14, 15);
+        assertAllFieldsGreaterThanZero(counts1);
+
+        DataCounts counts2 = new DataCounts(counts1);
+        assertAllFieldsGreaterThanZero(counts2);
+    }
+
+    private void assertAllFieldsEqualZero(DataCounts stats) throws Exception {
+        assertEquals(0L, stats.getProcessedRecordCount());
+        assertEquals(0L, stats.getProcessedFieldCount());
+        assertEquals(0L, stats.getInputBytes());
+        assertEquals(0L, stats.getInputFieldCount());
+        assertEquals(0L, stats.getInputRecordCount());
+        assertEquals(0L, stats.getInvalidDateCount());
+        assertEquals(0L, stats.getMissingFieldCount());
+        assertEquals(0L, stats.getOutOfOrderTimeStampCount());
+    }
+
+    private void assertAllFieldsGreaterThanZero(DataCounts stats) throws Exception {
+        assertThat(stats.getProcessedRecordCount(), greaterThan(0L));
+        assertThat(stats.getProcessedFieldCount(), greaterThan(0L));
+        assertThat(stats.getInputBytes(), greaterThan(0L));
+        assertThat(stats.getInputFieldCount(), greaterThan(0L));
+        assertThat(stats.getInputRecordCount(), greaterThan(0L));
+        assertThat(stats.getInputRecordCount(), greaterThan(0L));
+        assertThat(stats.getInvalidDateCount(), greaterThan(0L));
+        assertThat(stats.getMissingFieldCount(), greaterThan(0L));
+        assertThat(stats.getOutOfOrderTimeStampCount(), greaterThan(0L));
+        assertThat(stats.getLatestRecordTimeStamp().getTime(), greaterThan(0L));
+    }
+
+    private static DataCounts createCounts(
+            long processedRecordCount, long processedFieldCount, long inputBytes, long inputFieldCount,
+            long invalidDateCount, long missingFieldCount, long outOfOrderTimeStampCount, 
+            long emptyBucketCount, long sparseBucketCount, long bucketCount,
+            long earliestRecordTime, long latestRecordTime, long lastDataTimeStamp, long latestEmptyBucketTimeStamp,
+            long latestSparseBucketTimeStamp) {
+
+        DataCounts counts = new DataCounts("foo", processedRecordCount, processedFieldCount, inputBytes,
+                inputFieldCount, invalidDateCount, missingFieldCount, outOfOrderTimeStampCount,
+                emptyBucketCount, sparseBucketCount, bucketCount,
+                new Date(earliestRecordTime), new Date(latestRecordTime),
+                new Date(lastDataTimeStamp), new Date(latestEmptyBucketTimeStamp), new Date(latestSparseBucketTimeStamp));
+
+        return counts;
+    }
+
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStatsTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStatsTests.java
new file mode 100644
index 0000000000000..e3341123fb007
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStatsTests.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.process;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.protocol.xpack.ml.job.process.ModelSizeStats.MemoryStatus;
+
+import java.util.Date;
+
+public class ModelSizeStatsTests extends AbstractXContentTestCase<ModelSizeStats> {
+
+    public void testDefaultConstructor() {
+        ModelSizeStats stats = new ModelSizeStats.Builder("foo").build();
+        assertEquals(0, stats.getModelBytes());
+        assertEquals(0, stats.getTotalByFieldCount());
+        assertEquals(0, stats.getTotalOverFieldCount());
+        assertEquals(0, stats.getTotalPartitionFieldCount());
+        assertEquals(0, stats.getBucketAllocationFailuresCount());
+        assertEquals(MemoryStatus.OK, stats.getMemoryStatus());
+    }
+
+    public void testSetMemoryStatus_GivenNull() {
+        ModelSizeStats.Builder stats = new ModelSizeStats.Builder("foo");
+
+        NullPointerException ex = expectThrows(NullPointerException.class, () -> stats.setMemoryStatus(null));
+
+        assertEquals("[memory_status] must not be null", ex.getMessage());
+    }
+
+    public void testSetMemoryStatus_GivenSoftLimit() {
+        ModelSizeStats.Builder stats = new ModelSizeStats.Builder("foo");
+
+        stats.setMemoryStatus(MemoryStatus.SOFT_LIMIT);
+
+        assertEquals(MemoryStatus.SOFT_LIMIT, stats.build().getMemoryStatus());
+    }
+
+    @Override
+    protected ModelSizeStats createTestInstance() {
+        return createRandomized();
+    }
+
+    public static ModelSizeStats createRandomized() {
+        ModelSizeStats.Builder stats = new ModelSizeStats.Builder("foo");
+        if (randomBoolean()) {
+            stats.setBucketAllocationFailuresCount(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            stats.setModelBytes(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            stats.setTotalByFieldCount(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            stats.setTotalOverFieldCount(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            stats.setTotalPartitionFieldCount(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            stats.setLogTime(new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis()));
+        }
+        if (randomBoolean()) {
+            stats.setTimestamp(new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis()));
+        }
+        if (randomBoolean()) {
+            stats.setMemoryStatus(randomFrom(MemoryStatus.values()));
+        }
+        return stats.build();
+    }
+
+    @Override
+    protected ModelSizeStats doParseInstance(XContentParser parser) {
+        return ModelSizeStats.PARSER.apply(parser, null).build();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshotTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshotTests.java
new file mode 100644
index 0000000000000..8c6a9bd83c915
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshotTests.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.process;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.Date;
+
+public class ModelSnapshotTests extends AbstractXContentTestCase<ModelSnapshot> {
+
+    private static final Date DEFAULT_TIMESTAMP = new Date();
+    private static final String DEFAULT_DESCRIPTION = "a snapshot";
+    private static final String DEFAULT_ID = "my_id";
+    private static final int DEFAULT_DOC_COUNT = 7;
+    private static final Date DEFAULT_LATEST_RESULT_TIMESTAMP = new Date(12345678901234L);
+    private static final Date DEFAULT_LATEST_RECORD_TIMESTAMP = new Date(12345678904321L);
+    private static final boolean DEFAULT_RETAIN = true;
+
+    public void testCopyBuilder() {
+        ModelSnapshot modelSnapshot1 = createFullyPopulated().build();
+        ModelSnapshot modelSnapshot2 = new ModelSnapshot.Builder(modelSnapshot1).build();
+        assertEquals(modelSnapshot1, modelSnapshot2);
+    }
+
+    public void testEquals_GivenSameObject() {
+        ModelSnapshot modelSnapshot = createFullyPopulated().build();
+        assertTrue(modelSnapshot.equals(modelSnapshot));
+    }
+
+    public void testEquals_GivenObjectOfDifferentClass() {
+        ModelSnapshot modelSnapshot = createFullyPopulated().build();
+        assertFalse(modelSnapshot.equals("a string"));
+    }
+
+    public void testEquals_GivenEqualModelSnapshots() {
+        ModelSnapshot modelSnapshot1 = createFullyPopulated().build();
+        ModelSnapshot modelSnapshot2 = createFullyPopulated().build();
+
+        assertEquals(modelSnapshot1, modelSnapshot2);
+        assertEquals(modelSnapshot2, modelSnapshot1);
+        assertEquals(modelSnapshot1.hashCode(), modelSnapshot2.hashCode());
+    }
+
+    public void testEquals_GivenDifferentTimestamp() {
+        ModelSnapshot modelSnapshot1 = createFullyPopulated().build();
+        ModelSnapshot modelSnapshot2 = createFullyPopulated().setTimestamp(
+                new Date(modelSnapshot1.getTimestamp().getTime() + 1)).build();
+
+        assertFalse(modelSnapshot1.equals(modelSnapshot2));
+        assertFalse(modelSnapshot2.equals(modelSnapshot1));
+    }
+
+    public void testEquals_GivenDifferentDescription() {
+        ModelSnapshot modelSnapshot1 = createFullyPopulated().build();
+        ModelSnapshot modelSnapshot2 = createFullyPopulated()
+                .setDescription(modelSnapshot1.getDescription() + " blah").build();
+
+        assertFalse(modelSnapshot1.equals(modelSnapshot2));
+        assertFalse(modelSnapshot2.equals(modelSnapshot1));
+    }
+
+    public void testEquals_GivenDifferentId() {
+        ModelSnapshot modelSnapshot1 = createFullyPopulated().build();
+        ModelSnapshot modelSnapshot2 = createFullyPopulated()
+                .setSnapshotId(modelSnapshot1.getSnapshotId() + "_2").build();
+
+        assertFalse(modelSnapshot1.equals(modelSnapshot2));
+        assertFalse(modelSnapshot2.equals(modelSnapshot1));
+    }
+
+    public void testEquals_GivenDifferentDocCount() {
+        ModelSnapshot modelSnapshot1 = createFullyPopulated().build();
+        ModelSnapshot modelSnapshot2 = createFullyPopulated()
+                .setSnapshotDocCount(modelSnapshot1.getSnapshotDocCount() + 1).build();
+
+        assertFalse(modelSnapshot1.equals(modelSnapshot2));
+        assertFalse(modelSnapshot2.equals(modelSnapshot1));
+    }
+
+    public void testEquals_GivenDifferentModelSizeStats() {
+        ModelSnapshot modelSnapshot1 = createFullyPopulated().build();
+        ModelSizeStats.Builder modelSizeStats = new ModelSizeStats.Builder("foo");
+        modelSizeStats.setModelBytes(42L);
+        ModelSnapshot modelSnapshot2 = createFullyPopulated().setModelSizeStats(modelSizeStats).build();
+
+        assertFalse(modelSnapshot1.equals(modelSnapshot2));
+        assertFalse(modelSnapshot2.equals(modelSnapshot1));
+    }
+
+    public void testEquals_GivenDifferentQuantiles() {
+        ModelSnapshot modelSnapshot1 = createFullyPopulated().build();
+        ModelSnapshot modelSnapshot2 = createFullyPopulated()
+                .setQuantiles(new Quantiles("foo", modelSnapshot1.getQuantiles().getTimestamp(),
+                        "different state")).build();
+
+        assertFalse(modelSnapshot1.equals(modelSnapshot2));
+        assertFalse(modelSnapshot2.equals(modelSnapshot1));
+    }
+
+    public void testEquals_GivenDifferentLatestResultTimestamp() {
+        ModelSnapshot modelSnapshot1 = createFullyPopulated().build();
+        ModelSnapshot modelSnapshot2 = createFullyPopulated().setLatestResultTimeStamp(
+                new Date(modelSnapshot1.getLatestResultTimeStamp().getTime() + 1)).build();
+
+        assertFalse(modelSnapshot1.equals(modelSnapshot2));
+        assertFalse(modelSnapshot2.equals(modelSnapshot1));
+    }
+
+    public void testEquals_GivenDifferentLatestRecordTimestamp() {
+        ModelSnapshot modelSnapshot1 = createFullyPopulated().build();
+        ModelSnapshot modelSnapshot2 = createFullyPopulated().setLatestRecordTimeStamp(
+                new Date(modelSnapshot1.getLatestRecordTimeStamp().getTime() + 1)).build();
+
+        assertFalse(modelSnapshot1.equals(modelSnapshot2));
+        assertFalse(modelSnapshot2.equals(modelSnapshot1));
+    }
+
+    private static ModelSnapshot.Builder createFullyPopulated() {
+        ModelSnapshot.Builder modelSnapshot = new ModelSnapshot.Builder();
+        modelSnapshot.setJobId("foo");
+        modelSnapshot.setMinVersion(Version.CURRENT);
+        modelSnapshot.setTimestamp(DEFAULT_TIMESTAMP);
+        modelSnapshot.setDescription(DEFAULT_DESCRIPTION);
+        modelSnapshot.setSnapshotId(DEFAULT_ID);
+        modelSnapshot.setSnapshotDocCount(DEFAULT_DOC_COUNT);
+        ModelSizeStats.Builder modelSizeStatsBuilder = new ModelSizeStats.Builder("foo");
+        modelSizeStatsBuilder.setLogTime(null);
+        modelSnapshot.setModelSizeStats(modelSizeStatsBuilder);
+        modelSnapshot.setLatestResultTimeStamp(DEFAULT_LATEST_RESULT_TIMESTAMP);
+        modelSnapshot.setLatestRecordTimeStamp(DEFAULT_LATEST_RECORD_TIMESTAMP);
+        modelSnapshot.setQuantiles(new Quantiles("foo", DEFAULT_TIMESTAMP, "state"));
+        modelSnapshot.setRetain(DEFAULT_RETAIN);
+        return modelSnapshot;
+    }
+
+    @Override
+    protected ModelSnapshot createTestInstance() {
+        return createRandomized();
+    }
+
+    public static ModelSnapshot createRandomized() {
+        ModelSnapshot.Builder modelSnapshot = new ModelSnapshot.Builder(randomAlphaOfLengthBetween(1, 20));
+        modelSnapshot.setMinVersion(Version.CURRENT);
+        modelSnapshot.setTimestamp(new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis()));
+        modelSnapshot.setDescription(randomAlphaOfLengthBetween(1, 20));
+        modelSnapshot.setSnapshotId(randomAlphaOfLengthBetween(1, 20));
+        modelSnapshot.setSnapshotDocCount(randomInt());
+        modelSnapshot.setModelSizeStats(ModelSizeStatsTests.createRandomized());
+        modelSnapshot.setLatestResultTimeStamp(
+                new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis()));
+        modelSnapshot.setLatestRecordTimeStamp(
+                new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis()));
+        modelSnapshot.setQuantiles(QuantilesTests.createRandomized());
+        modelSnapshot.setRetain(randomBoolean());
+        return modelSnapshot.build();
+    }
+
+    @Override
+    protected ModelSnapshot doParseInstance(XContentParser parser){
+        return ModelSnapshot.PARSER.apply(parser, null).build();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/QuantilesTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/QuantilesTests.java
new file mode 100644
index 0000000000000..77ae21bc6f89a
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/QuantilesTests.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.process;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.Date;
+
+
+public class QuantilesTests extends AbstractXContentTestCase<Quantiles> {
+
+    public void testEquals_GivenSameObject() {
+        Quantiles quantiles = new Quantiles("foo", new Date(0L), "foo");
+        assertTrue(quantiles.equals(quantiles));
+    }
+
+
+    public void testEquals_GivenDifferentClassObject() {
+        Quantiles quantiles = new Quantiles("foo", new Date(0L), "foo");
+        assertFalse(quantiles.equals("not a quantiles object"));
+    }
+
+
+    public void testEquals_GivenEqualQuantilesObject() {
+        Quantiles quantiles1 = new Quantiles("foo", new Date(0L), "foo");
+
+        Quantiles quantiles2 = new Quantiles("foo", new Date(0L), "foo");
+
+        assertTrue(quantiles1.equals(quantiles2));
+        assertTrue(quantiles2.equals(quantiles1));
+    }
+
+
+    public void testEquals_GivenDifferentState() {
+        Quantiles quantiles1 = new Quantiles("foo", new Date(0L), "bar1");
+
+        Quantiles quantiles2 = new Quantiles("foo", new Date(0L), "bar2");
+
+        assertFalse(quantiles1.equals(quantiles2));
+        assertFalse(quantiles2.equals(quantiles1));
+    }
+
+
+    public void testHashCode_GivenEqualObject() {
+        Quantiles quantiles1 = new Quantiles("foo", new Date(0L), "foo");
+
+        Quantiles quantiles2 = new Quantiles("foo", new Date(0L), "foo");
+
+        assertEquals(quantiles1.hashCode(), quantiles2.hashCode());
+    }
+
+
+    @Override
+    protected Quantiles createTestInstance() {
+        return createRandomized();
+    }
+
+    public static Quantiles createRandomized() {
+        return new Quantiles(randomAlphaOfLengthBetween(1, 20),
+                new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis()),
+                randomAlphaOfLengthBetween(0, 1000));
+    }
+
+    @Override
+    protected Quantiles doParseInstance(XContentParser parser) {
+        return Quantiles.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyCauseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyCauseTests.java
new file mode 100644
index 0000000000000..070b9f18b4dc4
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyCauseTests.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AnomalyCauseTests extends AbstractXContentTestCase<AnomalyCause> {
+
+    @Override
+    protected AnomalyCause createTestInstance() {
+        AnomalyCause anomalyCause = new AnomalyCause();
+        if (randomBoolean()) {
+            int size = randomInt(10);
+            List<Double> actual = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                actual.add(randomDouble());
+            }
+            anomalyCause.setActual(actual);
+        }
+        if (randomBoolean()) {
+            int size = randomInt(10);
+            List<Double> typical = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                typical.add(randomDouble());
+            }
+            anomalyCause.setTypical(typical);
+        }
+        if (randomBoolean()) {
+            anomalyCause.setByFieldName(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            anomalyCause.setByFieldValue(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            anomalyCause.setCorrelatedByFieldValue(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            anomalyCause.setOverFieldName(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            anomalyCause.setOverFieldValue(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            anomalyCause.setPartitionFieldName(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            anomalyCause.setPartitionFieldValue(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            anomalyCause.setFunction(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            anomalyCause.setFunctionDescription(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            anomalyCause.setFieldName(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            anomalyCause.setProbability(randomDouble());
+        }
+        if (randomBoolean()) {
+            int size = randomInt(10);
+            List<Influence> influencers = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                int fieldValuesSize = randomInt(10);
+                List<String> fieldValues = new ArrayList<>(fieldValuesSize);
+                for (int j = 0; j < fieldValuesSize; j++) {
+                    fieldValues.add(randomAlphaOfLengthBetween(1, 20));
+                }
+                influencers.add(new Influence(randomAlphaOfLengthBetween(1, 20), fieldValues));
+            }
+            anomalyCause.setInfluencers(influencers);
+        }
+        return anomalyCause;
+    }
+
+    @Override
+    protected AnomalyCause doParseInstance(XContentParser parser) {
+        return AnomalyCause.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyRecordTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyRecordTests.java
new file mode 100644
index 0000000000000..d4cadb19796b5
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyRecordTests.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+
+public class AnomalyRecordTests extends AbstractXContentTestCase<AnomalyRecord> {
+
+    @Override
+    protected AnomalyRecord createTestInstance() {
+        return createTestInstance("foo");
+    }
+
+    public AnomalyRecord createTestInstance(String jobId) {
+        AnomalyRecord anomalyRecord = new AnomalyRecord(jobId, new Date(randomNonNegativeLong()), randomNonNegativeLong());
+        anomalyRecord.setActual(Collections.singletonList(randomDouble()));
+        anomalyRecord.setTypical(Collections.singletonList(randomDouble()));
+        anomalyRecord.setProbability(randomDouble());
+        anomalyRecord.setRecordScore(randomDouble());
+        anomalyRecord.setInitialRecordScore(randomDouble());
+        anomalyRecord.setInterim(randomBoolean());
+        if (randomBoolean()) {
+            anomalyRecord.setFieldName(randomAlphaOfLength(12));
+        }
+        if (randomBoolean()) {
+            anomalyRecord.setByFieldName(randomAlphaOfLength(12));
+            anomalyRecord.setByFieldValue(randomAlphaOfLength(12));
+        }
+        if (randomBoolean()) {
+            anomalyRecord.setPartitionFieldName(randomAlphaOfLength(12));
+            anomalyRecord.setPartitionFieldValue(randomAlphaOfLength(12));
+        }
+        if (randomBoolean()) {
+            anomalyRecord.setOverFieldName(randomAlphaOfLength(12));
+            anomalyRecord.setOverFieldValue(randomAlphaOfLength(12));
+        }
+        anomalyRecord.setFunction(randomAlphaOfLengthBetween(5, 20));
+        anomalyRecord.setFunctionDescription(randomAlphaOfLengthBetween(5, 20));
+        if (randomBoolean()) {
+            anomalyRecord.setCorrelatedByFieldValue(randomAlphaOfLength(16));
+        }
+        if (randomBoolean()) {
+            int count = randomIntBetween(0, 9);
+            List<Influence>  influences = new ArrayList<>();
+            for (int i=0; i<count; i++) {
+                influences.add(new Influence(randomAlphaOfLength(8), Collections.singletonList(randomAlphaOfLengthBetween(1, 28))));
+            }
+            anomalyRecord.setInfluencers(influences);
+        }
+        if (randomBoolean()) {
+            int count = randomIntBetween(0, 9);
+            List<AnomalyCause>  causes = new ArrayList<>();
+            for (int i=0; i<count; i++) {
+                causes.add(new AnomalyCauseTests().createTestInstance());
+            }
+            anomalyRecord.setCauses(causes);
+        }
+
+        return anomalyRecord;
+    }
+
+    @Override
+    protected AnomalyRecord doParseInstance(XContentParser parser) {
+        return AnomalyRecord.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketInfluencerTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketInfluencerTests.java
new file mode 100644
index 0000000000000..7e4c166d1fd67
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketInfluencerTests.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.Date;
+
+public class BucketInfluencerTests extends AbstractXContentTestCase<BucketInfluencer> {
+
+    @Override
+    protected BucketInfluencer createTestInstance() {
+        BucketInfluencer bucketInfluencer = new BucketInfluencer(randomAlphaOfLengthBetween(1, 20), new Date(randomNonNegativeLong()),
+                randomNonNegativeLong());
+        if (randomBoolean()) {
+            bucketInfluencer.setAnomalyScore(randomDouble());
+        }
+        if (randomBoolean()) {
+            bucketInfluencer.setInfluencerFieldName(randomAlphaOfLengthBetween(1, 20));
+        }
+        if (randomBoolean()) {
+            bucketInfluencer.setInitialAnomalyScore(randomDouble());
+        }
+        if (randomBoolean()) {
+            bucketInfluencer.setProbability(randomDouble());
+        }
+        if (randomBoolean()) {
+            bucketInfluencer.setRawAnomalyScore(randomDouble());
+        }
+        if (randomBoolean()) {
+            bucketInfluencer.setIsInterim(randomBoolean());
+        }
+        return bucketInfluencer;
+    }
+
+    @Override
+    protected BucketInfluencer doParseInstance(XContentParser parser) {
+        return BucketInfluencer.PARSER.apply(parser, null);
+    }
+
+    public void testEquals_GivenNull() {
+        assertFalse(new BucketInfluencer(randomAlphaOfLengthBetween(1, 20), new Date(), 600).equals(null));
+    }
+
+    public void testEquals_GivenDifferentClass() {
+        assertFalse(new BucketInfluencer(randomAlphaOfLengthBetween(1, 20), new Date(), 600).equals("a string"));
+    }
+
+    public void testEquals_GivenEqualInfluencers() {
+        BucketInfluencer bucketInfluencer1 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer1.setAnomalyScore(42.0);
+        bucketInfluencer1.setInfluencerFieldName("foo");
+        bucketInfluencer1.setInitialAnomalyScore(67.3);
+        bucketInfluencer1.setProbability(0.0003);
+        bucketInfluencer1.setRawAnomalyScore(3.14);
+
+        BucketInfluencer bucketInfluencer2 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer2.setAnomalyScore(42.0);
+        bucketInfluencer2.setInfluencerFieldName("foo");
+        bucketInfluencer2.setInitialAnomalyScore(67.3);
+        bucketInfluencer2.setProbability(0.0003);
+        bucketInfluencer2.setRawAnomalyScore(3.14);
+
+        assertTrue(bucketInfluencer1.equals(bucketInfluencer2));
+        assertTrue(bucketInfluencer2.equals(bucketInfluencer1));
+        assertEquals(bucketInfluencer1.hashCode(), bucketInfluencer2.hashCode());
+    }
+
+    public void testEquals_GivenDifferentAnomalyScore() {
+        BucketInfluencer bucketInfluencer1 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer1.setAnomalyScore(42.0);
+
+        BucketInfluencer bucketInfluencer2 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer2.setAnomalyScore(42.1);
+
+        assertFalse(bucketInfluencer1.equals(bucketInfluencer2));
+        assertFalse(bucketInfluencer2.equals(bucketInfluencer1));
+    }
+
+    public void testEquals_GivenDifferentFieldName() {
+        BucketInfluencer bucketInfluencer1 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer1.setInfluencerFieldName("foo");
+
+        BucketInfluencer bucketInfluencer2 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer2.setInfluencerFieldName("bar");
+
+        assertFalse(bucketInfluencer1.equals(bucketInfluencer2));
+        assertFalse(bucketInfluencer2.equals(bucketInfluencer1));
+    }
+
+    public void testEquals_GivenDifferentInitialAnomalyScore() {
+        BucketInfluencer bucketInfluencer1 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer1.setInitialAnomalyScore(42.0);
+
+        BucketInfluencer bucketInfluencer2 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer2.setInitialAnomalyScore(42.1);
+
+        assertFalse(bucketInfluencer1.equals(bucketInfluencer2));
+        assertFalse(bucketInfluencer2.equals(bucketInfluencer1));
+    }
+
+    public void testEquals_GivenRawAnomalyScore() {
+        BucketInfluencer bucketInfluencer1 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer1.setRawAnomalyScore(42.0);
+
+        BucketInfluencer bucketInfluencer2 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer2.setRawAnomalyScore(42.1);
+
+        assertFalse(bucketInfluencer1.equals(bucketInfluencer2));
+        assertFalse(bucketInfluencer2.equals(bucketInfluencer1));
+    }
+
+    public void testEquals_GivenDifferentProbability() {
+        BucketInfluencer bucketInfluencer1 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer1.setProbability(0.001);
+
+        BucketInfluencer bucketInfluencer2 = new BucketInfluencer("foo", new Date(123), 600);
+        bucketInfluencer2.setProbability(0.002);
+
+        assertFalse(bucketInfluencer1.equals(bucketInfluencer2));
+        assertFalse(bucketInfluencer2.equals(bucketInfluencer1));
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketTests.java
new file mode 100644
index 0000000000000..28b1893afe18b
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketTests.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.IntStream;
+
+public class BucketTests extends AbstractXContentTestCase<Bucket> {
+
+    @Override
+    public Bucket createTestInstance() {
+        return createTestInstance("foo");
+    }
+
+    public Bucket createTestInstance(String jobId) {
+        Bucket bucket = new Bucket(jobId, new Date(randomNonNegativeLong()), randomNonNegativeLong());
+        if (randomBoolean()) {
+            bucket.setAnomalyScore(randomDouble());
+        }
+        if (randomBoolean()) {
+            int size = randomInt(10);
+            List<BucketInfluencer> bucketInfluencers = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                BucketInfluencer bucketInfluencer = new BucketInfluencer(jobId, new Date(), 600);
+                bucketInfluencer.setAnomalyScore(randomDouble());
+                bucketInfluencer.setInfluencerFieldName(randomAlphaOfLengthBetween(1, 20));
+                bucketInfluencer.setInitialAnomalyScore(randomDouble());
+                bucketInfluencer.setProbability(randomDouble());
+                bucketInfluencer.setRawAnomalyScore(randomDouble());
+                bucketInfluencers.add(bucketInfluencer);
+            }
+            bucket.setBucketInfluencers(bucketInfluencers);
+        }
+        if (randomBoolean()) {
+            bucket.setEventCount(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            bucket.setInitialAnomalyScore(randomDouble());
+        }
+        if (randomBoolean()) {
+            bucket.setInterim(randomBoolean());
+        }
+        if (randomBoolean()) {
+            bucket.setProcessingTimeMs(randomLong());
+        }
+        if (randomBoolean()) {
+            int size = randomInt(10);
+            List<AnomalyRecord> records = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                AnomalyRecord anomalyRecord = new AnomalyRecordTests().createTestInstance(jobId);
+                records.add(anomalyRecord);
+            }
+            bucket.setRecords(records);
+        }
+        if (randomBoolean()) {
+            int size = randomInt(10);
+            List<String> scheduledEvents = new ArrayList<>(size);
+            IntStream.range(0, size).forEach(i -> scheduledEvents.add(randomAlphaOfLength(20)));
+            bucket.setScheduledEvents(scheduledEvents);
+        }
+        return bucket;
+    }
+
+    @Override
+    protected Bucket doParseInstance(XContentParser parser) {
+        return Bucket.PARSER.apply(parser, null);
+    }
+
+    public void testEquals_GivenDifferentClass() {
+        Bucket bucket = new Bucket("foo", new Date(randomLong()), randomNonNegativeLong());
+        assertFalse(bucket.equals("a string"));
+    }
+
+    public void testEquals_GivenTwoDefaultBuckets() {
+        Bucket bucket1 = new Bucket("foo", new Date(123), 123);
+        Bucket bucket2 = new Bucket("foo", new Date(123), 123);
+
+        assertTrue(bucket1.equals(bucket2));
+        assertTrue(bucket2.equals(bucket1));
+    }
+
+    public void testEquals_GivenDifferentAnomalyScore() {
+        Bucket bucket1 = new Bucket("foo", new Date(123), 123);
+        bucket1.setAnomalyScore(3.0);
+        Bucket bucket2 = new Bucket("foo", new Date(123), 123);
+        bucket2.setAnomalyScore(2.0);
+
+        assertFalse(bucket1.equals(bucket2));
+        assertFalse(bucket2.equals(bucket1));
+    }
+
+    public void testEquals_GivenSameDates() {
+        Bucket b1 = new Bucket("foo", new Date(1234567890L), 123);
+        Bucket b2 = new Bucket("foo", new Date(1234567890L), 123);
+        assertTrue(b1.equals(b2));
+    }
+
+    public void testEquals_GivenDifferentEventCount() {
+        Bucket bucket1 = new Bucket("foo", new Date(123), 123);
+        bucket1.setEventCount(3);
+        Bucket bucket2 = new Bucket("foo", new Date(123), 123);
+        bucket2.setEventCount(100);
+
+        assertFalse(bucket1.equals(bucket2));
+        assertFalse(bucket2.equals(bucket1));
+    }
+
+    public void testEquals_GivenOneHasRecordsAndTheOtherDoesNot() {
+        Bucket bucket1 = new Bucket("foo", new Date(123), 123);
+        bucket1.setRecords(Collections.singletonList(new AnomalyRecord("foo", new Date(123), 123)));
+        Bucket bucket2 = new Bucket("foo", new Date(123), 123);
+        bucket2.setRecords(Collections.emptyList());
+
+        assertFalse(bucket1.equals(bucket2));
+        assertFalse(bucket2.equals(bucket1));
+    }
+
+    public void testEquals_GivenDifferentNumberOfRecords() {
+        Bucket bucket1 = new Bucket("foo", new Date(123), 123);
+        bucket1.setRecords(Collections.singletonList(new AnomalyRecord("foo", new Date(123), 123)));
+        Bucket bucket2 = new Bucket("foo", new Date(123), 123);
+        bucket2.setRecords(Arrays.asList(new AnomalyRecord("foo", new Date(123), 123),
+                new AnomalyRecord("foo", new Date(123), 123)));
+
+        assertFalse(bucket1.equals(bucket2));
+        assertFalse(bucket2.equals(bucket1));
+    }
+
+    public void testEquals_GivenSameNumberOfRecordsButDifferent() {
+        AnomalyRecord anomalyRecord1 = new AnomalyRecord("foo", new Date(123), 123);
+        anomalyRecord1.setRecordScore(1.0);
+        AnomalyRecord anomalyRecord2 = new AnomalyRecord("foo", new Date(123), 123);
+        anomalyRecord1.setRecordScore(2.0);
+
+        Bucket bucket1 = new Bucket("foo", new Date(123), 123);
+        bucket1.setRecords(Collections.singletonList(anomalyRecord1));
+        Bucket bucket2 = new Bucket("foo", new Date(123), 123);
+        bucket2.setRecords(Collections.singletonList(anomalyRecord2));
+
+        assertFalse(bucket1.equals(bucket2));
+        assertFalse(bucket2.equals(bucket1));
+    }
+
+    public void testEquals_GivenDifferentIsInterim() {
+        Bucket bucket1 = new Bucket("foo", new Date(123), 123);
+        bucket1.setInterim(true);
+        Bucket bucket2 = new Bucket("foo", new Date(123), 123);
+        bucket2.setInterim(false);
+
+        assertFalse(bucket1.equals(bucket2));
+        assertFalse(bucket2.equals(bucket1));
+    }
+
+    public void testEquals_GivenDifferentBucketInfluencers() {
+        Bucket bucket1 = new Bucket("foo", new Date(123), 123);
+        BucketInfluencer influencer1 = new BucketInfluencer("foo", new Date(123), 123);
+        influencer1.setInfluencerFieldName("foo");
+        bucket1.setBucketInfluencers(Collections.singletonList(influencer1));
+
+        Bucket bucket2 = new Bucket("foo", new Date(123), 123);
+        BucketInfluencer influencer2 = new BucketInfluencer("foo", new Date(123), 123);
+        influencer2.setInfluencerFieldName("bar");
+        bucket2.setBucketInfluencers(Collections.singletonList(influencer2));
+
+        assertFalse(bucket1.equals(bucket2));
+        assertFalse(bucket2.equals(bucket1));
+    }
+
+    public void testEquals_GivenEqualBuckets() {
+        AnomalyRecord record = new AnomalyRecord("job_id", new Date(123), 123);
+        BucketInfluencer bucketInfluencer = new BucketInfluencer("foo", new Date(123), 123);
+        Date date = new Date();
+
+        Bucket bucket1 = new Bucket("foo", date, 123);
+        bucket1.setAnomalyScore(42.0);
+        bucket1.setInitialAnomalyScore(92.0);
+        bucket1.setEventCount(134);
+        bucket1.setInterim(true);
+        bucket1.setRecords(Collections.singletonList(record));
+        bucket1.setBucketInfluencers(Collections.singletonList(bucketInfluencer));
+
+        Bucket bucket2 = new Bucket("foo", date, 123);
+        bucket2.setAnomalyScore(42.0);
+        bucket2.setInitialAnomalyScore(92.0);
+        bucket2.setEventCount(134);
+        bucket2.setInterim(true);
+        bucket2.setRecords(Collections.singletonList(record));
+        bucket2.setBucketInfluencers(Collections.singletonList(bucketInfluencer));
+
+        assertTrue(bucket1.equals(bucket2));
+        assertTrue(bucket2.equals(bucket1));
+        assertEquals(bucket1.hashCode(), bucket2.hashCode());
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/CategoryDefinitionTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/CategoryDefinitionTests.java
new file mode 100644
index 0000000000000..28ef4a5ecb241
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/CategoryDefinitionTests.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.Arrays;
+
+public class CategoryDefinitionTests extends AbstractXContentTestCase<CategoryDefinition> {
+
+    public CategoryDefinition createTestInstance(String jobId) {
+        CategoryDefinition categoryDefinition = new CategoryDefinition(jobId);
+        categoryDefinition.setCategoryId(randomLong());
+        categoryDefinition.setTerms(randomAlphaOfLength(10));
+        categoryDefinition.setRegex(randomAlphaOfLength(10));
+        categoryDefinition.setMaxMatchingLength(randomLong());
+        categoryDefinition.setExamples(Arrays.asList(generateRandomStringArray(10, 10, false)));
+        if (randomBoolean()) {
+            categoryDefinition.setGrokPattern(randomAlphaOfLength(50));
+        }
+        return categoryDefinition;
+    }
+
+    @Override
+    protected CategoryDefinition createTestInstance() {
+        return createTestInstance(randomAlphaOfLength(10));
+    }
+
+    @Override
+    protected CategoryDefinition doParseInstance(XContentParser parser) {
+        return CategoryDefinition.PARSER.apply(parser, null);
+    }
+
+    public void testEquals_GivenSameObject() {
+        CategoryDefinition category = new CategoryDefinition(randomAlphaOfLength(10));
+
+        assertTrue(category.equals(category));
+    }
+
+    public void testEquals_GivenObjectOfDifferentClass() {
+        CategoryDefinition category = new CategoryDefinition(randomAlphaOfLength(10));
+
+        assertFalse(category.equals("a string"));
+    }
+
+    public void testEquals_GivenEqualCategoryDefinitions() {
+        CategoryDefinition category1 = createFullyPopulatedCategoryDefinition();
+        CategoryDefinition category2 = createFullyPopulatedCategoryDefinition();
+
+        assertTrue(category1.equals(category2));
+        assertTrue(category2.equals(category1));
+        assertEquals(category1.hashCode(), category2.hashCode());
+    }
+
+    public void testEquals_GivenCategoryDefinitionsWithDifferentIds() {
+        CategoryDefinition category1 = createFullyPopulatedCategoryDefinition();
+        CategoryDefinition category2 = createFullyPopulatedCategoryDefinition();
+        category2.setCategoryId(category1.getCategoryId() + 1);
+
+        assertFalse(category1.equals(category2));
+        assertFalse(category2.equals(category1));
+    }
+
+    public void testEquals_GivenCategoryDefinitionsWithDifferentTerms() {
+        CategoryDefinition category1 = createFullyPopulatedCategoryDefinition();
+        CategoryDefinition category2 = createFullyPopulatedCategoryDefinition();
+        category2.setTerms(category1.getTerms() + " additional");
+
+        assertFalse(category1.equals(category2));
+        assertFalse(category2.equals(category1));
+    }
+
+    public void testEquals_GivenCategoryDefinitionsWithDifferentRegex() {
+        CategoryDefinition category1 = createFullyPopulatedCategoryDefinition();
+        CategoryDefinition category2 = createFullyPopulatedCategoryDefinition();
+        category2.setRegex(category1.getRegex() + ".*additional.*");
+
+        assertFalse(category1.equals(category2));
+        assertFalse(category2.equals(category1));
+    }
+
+    public void testEquals_GivenCategoryDefinitionsWithDifferentMaxMatchingLength() {
+        CategoryDefinition category1 = createFullyPopulatedCategoryDefinition();
+        CategoryDefinition category2 = createFullyPopulatedCategoryDefinition();
+        category2.setMaxMatchingLength(category1.getMaxMatchingLength() + 1);
+
+        assertFalse(category1.equals(category2));
+        assertFalse(category2.equals(category1));
+    }
+
+    public void testEquals_GivenCategoryDefinitionsWithDifferentExamples() {
+        CategoryDefinition category1 = createFullyPopulatedCategoryDefinition();
+        CategoryDefinition category2 = createFullyPopulatedCategoryDefinition();
+        category2.addExample("additional");
+
+        assertFalse(category1.equals(category2));
+        assertFalse(category2.equals(category1));
+    }
+
+    private static CategoryDefinition createFullyPopulatedCategoryDefinition() {
+        CategoryDefinition category = new CategoryDefinition("jobName");
+        category.setCategoryId(42);
+        category.setTerms("foo bar");
+        category.setRegex(".*?foo.*?bar.*");
+        category.setMaxMatchingLength(120L);
+        category.addExample("foo");
+        category.addExample("bar");
+        return category;
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/InfluenceTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/InfluenceTests.java
new file mode 100644
index 0000000000000..b029997d015fa
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/InfluenceTests.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class InfluenceTests extends AbstractXContentTestCase<Influence> {
+
+    @Override
+    protected Influence createTestInstance() {
+        int size = randomInt(10);
+        List<String> fieldValues = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            fieldValues.add(randomAlphaOfLengthBetween(1, 20));
+        }
+        return new Influence(randomAlphaOfLengthBetween(1, 30), fieldValues);
+    }
+
+    @Override
+    protected Influence doParseInstance(XContentParser parser) {
+        return Influence.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/InfluencerTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/InfluencerTests.java
new file mode 100644
index 0000000000000..8125a1a5c725f
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/InfluencerTests.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.Date;
+
+public class InfluencerTests extends AbstractXContentTestCase<Influencer> {
+
+    public Influencer createTestInstance(String jobId) {
+        Influencer influencer = new Influencer(jobId, randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20),
+                new Date(randomNonNegativeLong()), randomNonNegativeLong());
+        influencer.setInterim(randomBoolean());
+        influencer.setInfluencerScore(randomDouble());
+        influencer.setInitialInfluencerScore(randomDouble());
+        influencer.setProbability(randomDouble());
+        return influencer;
+    }
+
+    @Override
+    protected Influencer createTestInstance() {
+        return createTestInstance(randomAlphaOfLengthBetween(1, 20));
+    }
+
+    @Override
+    protected Influencer doParseInstance(XContentParser parser) {
+        return Influencer.PARSER.apply(parser, null);
+    }
+
+    public void testToXContentDoesNotIncludeNameValueFieldWhenReservedWord() throws IOException {
+        Influencer influencer = new Influencer("foo", Influencer.INFLUENCER_SCORE.getPreferredName(), "bar", new Date(), 300L);
+        BytesReference bytes = XContentHelper.toXContent(influencer, XContentType.JSON, false);
+        XContentParser parser = createParser(XContentType.JSON.xContent(), bytes);
+        Object serialisedFieldValue = parser.map().get(Influencer.INFLUENCER_SCORE.getPreferredName());
+        assertNotNull(serialisedFieldValue);
+        assertNotEquals("bar", serialisedFieldValue);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/OverallBucketTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/OverallBucketTests.java
new file mode 100644
index 0000000000000..babd7410d57ce
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/results/OverallBucketTests.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.results;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.lessThan;
+
+public class OverallBucketTests extends AbstractXContentTestCase<OverallBucket> {
+
+    @Override
+    protected OverallBucket createTestInstance() {
+        int jobCount = randomIntBetween(0, 10);
+        List<OverallBucket.JobInfo> jobs = new ArrayList<>(jobCount);
+        for (int i = 0; i < jobCount; ++i) {
+            jobs.add(new OverallBucket.JobInfo(randomAlphaOfLength(10), randomDoubleBetween(0.0, 100.0, true)));
+        }
+        OverallBucket overallBucket = new OverallBucket(new Date(randomNonNegativeLong()),
+                randomIntBetween(60, 24 * 3600),
+                randomDoubleBetween(0.0, 100.0, true),
+                randomBoolean());
+        overallBucket.setJobs(jobs);
+        return overallBucket;
+    }
+
+    public void testCompareTo() {
+        OverallBucket.JobInfo jobInfo1 = new OverallBucket.JobInfo("aaa", 1.0);
+        OverallBucket.JobInfo jobInfo2 = new OverallBucket.JobInfo("aaa", 3.0);
+        OverallBucket.JobInfo jobInfo3 = new OverallBucket.JobInfo("bbb", 1.0);
+        assertThat(jobInfo1.compareTo(jobInfo1), equalTo(0));
+        assertThat(jobInfo1.compareTo(jobInfo2), lessThan(0));
+        assertThat(jobInfo1.compareTo(jobInfo3), lessThan(0));
+        assertThat(jobInfo2.compareTo(jobInfo3), lessThan(0));
+    }
+
+    @Override
+    protected OverallBucket doParseInstance(XContentParser parser) {
+        return OverallBucket.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/security/UserTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/security/UserTests.java
new file mode 100644
index 0000000000000..2e3c67131df74
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/security/UserTests.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.protocol.xpack.security;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.is;
+
+public class UserTests extends ESTestCase {
+
+    public void testUserToString() {
+        User user = new User("u1", "r1");
+        assertThat(user.toString(), is("User[username=u1,roles=[r1],fullName=null,email=null,metadata={}]"));
+        user = new User("u1", new String[] { "r1", "r2" }, "user1", "user1@domain.com", Collections.singletonMap("key", "val"), true);
+        assertThat(user.toString(), is("User[username=u1,roles=[r1,r2],fullName=user1,email=user1@domain.com,metadata={key=val}]"));
+        user = new User("u1", new String[] {"r1"}, new User("u2", "r2", "r3"));
+        assertThat(user.toString(), is("User[username=u1,roles=[r1],fullName=null,email=null,metadata={}," +
+            "authenticatedUser=[User[username=u2,roles=[r2,r3],fullName=null,email=null,metadata={}]]]"));
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponseTests.java
new file mode 100644
index 0000000000000..1dbc4cec32128
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponseTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.watcher;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class DeleteWatchResponseTests extends AbstractXContentTestCase<DeleteWatchResponse> {
+
+    @Override
+    protected DeleteWatchResponse createTestInstance() {
+        String id = randomAlphaOfLength(10);
+        long version = randomLongBetween(1, 10);
+        boolean found = randomBoolean();
+        return new DeleteWatchResponse(id, version, found);
+    }
+
+    @Override
+    protected DeleteWatchResponse doParseInstance(XContentParser parser) throws IOException {
+        return DeleteWatchResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
index b91092fe9e750..24303b8342b7e 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -32,6 +32,7 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
 import java.util.Locale;
@@ -41,6 +42,7 @@
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.everyItem;
@@ -323,6 +325,115 @@ public void testRollupAfterRestart() throws Exception {
         }
     }
 
+    public void testRollupIDSchemeAfterRestart() throws Exception {
+        assumeTrue("Rollup can be tested with 6.3.0 and onwards", oldClusterVersion.onOrAfter(Version.V_6_3_0));
+        assumeTrue("Rollup ID scheme changed in 6.4", oldClusterVersion.before(Version.V_6_4_0));
+        if (runningAgainstOldCluster) {
+
+            final Request indexRequest = new Request("POST", "/id-test-rollup/_doc/1");
+            indexRequest.setJsonEntity("{\"timestamp\":\"2018-01-01T00:00:01\",\"value\":123}");
+            client().performRequest(indexRequest);
+
+            // create the rollup job
+            final Request createRollupJobRequest = new Request("PUT", "/_xpack/rollup/job/rollup-id-test");
+            createRollupJobRequest.setJsonEntity("{"
+                + "\"index_pattern\":\"id-test-rollup\","
+                + "\"rollup_index\":\"id-test-results-rollup\","
+                + "\"cron\":\"*/1 * * * * ?\","
+                + "\"page_size\":100,"
+                + "\"groups\":{"
+                + "    \"date_histogram\":{"
+                + "        \"field\":\"timestamp\","
+                + "        \"interval\":\"5m\""
+                + "      },"
+                +       "\"histogram\":{"
+                + "        \"fields\": [\"value\"],"
+                + "        \"interval\":1"
+                + "      },"
+                +       "\"terms\":{"
+                + "        \"fields\": [\"value\"]"
+                + "      }"
+                + "},"
+                + "\"metrics\":["
+                + "    {\"field\":\"value\",\"metrics\":[\"min\",\"max\",\"sum\"]}"
+                + "]"
+                + "}");
+
+            Map<String, Object> createRollupJobResponse = entityAsMap(client().performRequest(createRollupJobRequest));
+            assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+            // start the rollup job
+            final Request startRollupJobRequest = new Request("POST", "_xpack/rollup/job/rollup-id-test/_start");
+            Map<String, Object> startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest));
+            assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE));
+
+            assertRollUpJob("rollup-id-test");
+
+            assertBusy(() -> {
+                client().performRequest(new Request("POST", "id-test-results-rollup/_refresh"));
+                final Request searchRequest = new Request("GET", "id-test-results-rollup/_search");
+                try {
+                    Map<String, Object> searchResponse = entityAsMap(client().performRequest(searchRequest));
+                    assertNotNull(ObjectPath.eval("hits.total", searchResponse));
+                    assertThat(ObjectPath.eval("hits.total", searchResponse), equalTo(1));
+                    assertThat(ObjectPath.eval("hits.hits.0._id", searchResponse), equalTo("3310683722"));
+                } catch (IOException e) {
+                    fail();
+                }
+            });
+
+            // After we've confirmed the doc, wait until we move back to STARTED so that we know the
+            // state was saved at the end
+            waitForRollUpJob("rollup-id-test", equalTo("started"));
+
+        } else {
+
+            final Request indexRequest = new Request("POST", "/id-test-rollup/_doc/2");
+            indexRequest.setJsonEntity("{\"timestamp\":\"2018-01-02T00:00:01\",\"value\":345}");
+            client().performRequest(indexRequest);
+
+            // stop the rollup job to force a state save, which will upgrade the ID
+            final Request stopRollupJobRequest = new Request("POST", "_xpack/rollup/job/rollup-id-test/_stop");
+            Map<String, Object> stopRollupJobResponse = entityAsMap(client().performRequest(stopRollupJobRequest));
+            assertThat(stopRollupJobResponse.get("stopped"), equalTo(Boolean.TRUE));
+
+            waitForRollUpJob("rollup-id-test", equalTo("stopped"));
+
+            // start the rollup job again
+            final Request startRollupJobRequest = new Request("POST", "_xpack/rollup/job/rollup-id-test/_start");
+            Map<String, Object> startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest));
+            assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE));
+
+            waitForRollUpJob("rollup-id-test", anyOf(equalTo("indexing"), equalTo("started")));
+
+            assertBusy(() -> {
+                client().performRequest(new Request("POST", "id-test-results-rollup/_refresh"));
+                final Request searchRequest = new Request("GET", "id-test-results-rollup/_search");
+                try {
+                    Map<String, Object> searchResponse = entityAsMap(client().performRequest(searchRequest));
+                    assertNotNull(ObjectPath.eval("hits.total", searchResponse));
+                    assertThat(ObjectPath.eval("hits.total", searchResponse), equalTo(2));
+                    List<String> ids = new ArrayList<>(2);
+                    ids.add(ObjectPath.eval("hits.hits.0._id", searchResponse));
+                    ids.add(ObjectPath.eval("hits.hits.1._id", searchResponse));
+
+                    // should have both old and new ID formats
+                    assertThat(ids, containsInAnyOrder("3310683722", "rollup-id-test$ehY4NAyVSy8xxUDZrNXXIA"));
+
+                    List<Double> values = new ArrayList<>(2);
+                    Map<String, Object> doc = ObjectPath.eval("hits.hits.0._source", searchResponse);
+                    values.add((Double)doc.get("value.min.value"));
+                    doc = ObjectPath.eval("hits.hits.1._source", searchResponse);
+                    values.add((Double)doc.get("value.min.value"));
+
+                    assertThat(values, containsInAnyOrder(123.0, 345.0));
+                } catch (IOException e) {
+                    fail();
+                }
+            });
+        }
+    }
+
     public void testSqlFailsOnIndexWithTwoTypes() throws IOException {
         // TODO this isn't going to trigger until we backport to 6.1
         assumeTrue("It is only possible to build an index that sql doesn't like before 6.0.0",
diff --git a/x-pack/qa/full-cluster-restart/with-system-key/build.gradle b/x-pack/qa/full-cluster-restart/with-system-key/build.gradle
index e69de29bb2d1d..928280b6584bd 100644
--- a/x-pack/qa/full-cluster-restart/with-system-key/build.gradle
+++ b/x-pack/qa/full-cluster-restart/with-system-key/build.gradle
@@ -0,0 +1,8 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+// Skip test on FIPS FIXME https://github.com/elastic/elasticsearch/issues/32737
+if (project.inFipsJvm) {
+    tasks.withType(RestIntegTestTask) {
+        enabled = false
+    }
+}
diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle
index d2818bfc127e5..59667d9ee7809 100644
--- a/x-pack/qa/kerberos-tests/build.gradle
+++ b/x-pack/qa/kerberos-tests/build.gradle
@@ -36,32 +36,43 @@ task krb5kdcFixture(type: org.elasticsearch.gradle.test.VagrantFixture) {
     dependsOn krb5kdcUpdate
 }
 
-task krb5AddPrincipals { dependsOn krb5kdcFixture }
+// lazily resolve to avoid any slowdowns from DNS lookups prior to when we need this value
+Object httpPrincipal = new Object() {
+    @Override
+    String toString() {
+        InetAddress resolvedAddress = InetAddress.getByName('127.0.0.1')
+        return "HTTP/" + resolvedAddress.getCanonicalHostName()
+    }
+}
 
-List<String> principals = [
-    "HTTP/localhost",
-    "peppa",
-    "george~dino"
-]
 String realm = "BUILD.ELASTIC.CO"
 
-for (String principal : principals) {
-    String[] princPwdPair = principal.split('~');
-    String princName = princPwdPair[0];
-    String password = "";
-    if (princPwdPair.length > 1) {
-        password = princPwdPair[1];
-    }
-    Task create = project.tasks.create("addPrincipal#${principal}".replace('/', '_'), org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
-        command 'ssh'
-        args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh $princName $password"
-        boxName box
-        environmentVars vagrantEnvVars
-        dependsOn krb5kdcFixture
-    }
-    krb5AddPrincipals.dependsOn(create)
+task 'addPrincipal#peppa'(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'ssh'
+    args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh peppa "
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn krb5kdcFixture
+}
+
+task 'addPrincipal#george'(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'ssh'
+    args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh george dino"
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn krb5kdcFixture
+}
+
+task 'addPrincipal#HTTP'(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'ssh'
+    args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh $httpPrincipal"
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn krb5kdcFixture
 }
 
+task krb5AddPrincipals { dependsOn krb5kdcFixture, 'addPrincipal#peppa', 'addPrincipal#george', 'addPrincipal#HTTP' }
+
 def generatedResources = "$buildDir/generated-resources/keytabs"
 task copyKeytabToGeneratedResources(type: Copy) {
     Path peppaKeytab = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("peppa.keytab").toAbsolutePath()
@@ -71,6 +82,9 @@ task copyKeytabToGeneratedResources(type: Copy) {
 }
 
 integTestCluster {
+    // force localhost IPv4 otherwise it is a chicken and egg problem where we need the keytab for the hostname when starting the cluster
+    // but do not know the exact address that is first in the http ports file
+    setting 'http.host', '127.0.0.1'
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.security.authc.realms.file.type', 'file'
@@ -87,7 +101,8 @@ integTestCluster {
     Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
     String jvmArgsStr = " -Djava.security.krb5.conf=${krb5conf}" + " -Dsun.security.krb5.debug=true"
     jvmArgs jvmArgsStr
-    Path esKeytab = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("HTTP_localhost.keytab").toAbsolutePath()
+    Path esKeytab = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs")
+            .resolve("$httpPrincipal".replace('/', '_') + ".keytab").toAbsolutePath()
     extraConfigFile("es.keytab", "${esKeytab}")
 
     setupCommand 'setupTestAdmin',
diff --git a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
index d5928cb58f687..b6ebfde20799f 100644
--- a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
+++ b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -23,6 +24,8 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.security.AccessControlContext;
 import java.security.AccessController;
 import java.security.PrivilegedActionException;
@@ -100,6 +103,22 @@ public void testLoginByUsernamePassword() throws IOException, PrivilegedActionEx
         executeRequestAndVerifyResponse(userPrincipalName, callbackHandler);
     }
 
+    public void testSoDoesNotFailWithNoTests() {
+        // intentionally empty - this is just needed to ensure the build does not fail because we mute its only test.
+    }
+
+    @Override
+    @SuppressForbidden(reason = "SPNEGO relies on hostnames and we need to ensure host isn't a IP address")
+    protected HttpHost buildHttpHost(String host, int port) {
+        try {
+            InetAddress inetAddress = InetAddress.getByName(host);
+            return super.buildHttpHost(inetAddress.getCanonicalHostName(), port);
+        } catch (UnknownHostException e) {
+            assumeNoException("failed to resolve host [" + host + "]", e);
+        }
+        throw new IllegalStateException("DNS not resolved and assume did not trip");
+    }
+
     private void executeRequestAndVerifyResponse(final String userPrincipalName,
             final SpnegoHttpClientConfigCallbackHandler callbackHandler) throws PrivilegedActionException, IOException {
         final Request request = new Request("GET", "/_xpack/security/_authenticate");
diff --git a/x-pack/qa/kerberos-tests/src/test/resources/plugin-security.policy b/x-pack/qa/kerberos-tests/src/test/resources/plugin-security.policy
index fb7936bf62093..84219494bf2ce 100644
--- a/x-pack/qa/kerberos-tests/src/test/resources/plugin-security.policy
+++ b/x-pack/qa/kerberos-tests/src/test/resources/plugin-security.policy
@@ -1,4 +1,7 @@
 grant {
   permission javax.security.auth.AuthPermission "doAsPrivileged";
   permission javax.security.auth.kerberos.DelegationPermission "\"HTTP/localhost@BUILD.ELASTIC.CO\" \"krbtgt/BUILD.ELASTIC.CO@BUILD.ELASTIC.CO\"";
+  permission javax.security.auth.kerberos.DelegationPermission "\"HTTP/localhost.localdomain@BUILD.ELASTIC.CO\" \"krbtgt/BUILD.ELASTIC.CO@BUILD.ELASTIC.CO\"";
+  permission javax.security.auth.kerberos.DelegationPermission "\"HTTP/localhost4@BUILD.ELASTIC.CO\" \"krbtgt/BUILD.ELASTIC.CO@BUILD.ELASTIC.CO\"";
+  permission javax.security.auth.kerberos.DelegationPermission "\"HTTP/localhost4.localdomain4@BUILD.ELASTIC.CO\" \"krbtgt/BUILD.ELASTIC.CO@BUILD.ELASTIC.CO\"";
 };
\ No newline at end of file
diff --git a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ReopenJobResetsFinishedTimeIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ReopenJobResetsFinishedTimeIT.java
new file mode 100644
index 0000000000000..325b1370315ca
--- /dev/null
+++ b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ReopenJobResetsFinishedTimeIT.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.integration;
+
+import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
+import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
+import org.elasticsearch.xpack.core.ml.job.config.Detector;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.junit.After;
+
+import java.util.Collections;
+
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.Matchers.is;
+
+public class ReopenJobResetsFinishedTimeIT extends MlNativeAutodetectIntegTestCase {
+
+    @After
+    public void cleanUpTest() {
+        cleanUp();
+    }
+
+    public void test() {
+        final String jobId = "reset-finished-time-test";
+        Job.Builder job = createJob(jobId);
+
+        registerJob(job);
+        putJob(job);
+        openJob(job.getId());
+
+        assertThat(getSingleJob(jobId).getFinishedTime(), is(nullValue()));
+
+        closeJob(jobId);
+        assertThat(getSingleJob(jobId).getFinishedTime(), is(notNullValue()));
+
+        openJob(jobId);
+        assertThat(getSingleJob(jobId).getFinishedTime(), is(nullValue()));
+    }
+
+    private Job getSingleJob(String jobId) {
+        return getJob(jobId).get(0);
+    }
+
+    private Job.Builder createJob(String id) {
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setFormat(DataDescription.DataFormat.XCONTENT);
+        dataDescription.setTimeFormat(DataDescription.EPOCH_MS);
+
+        Detector.Builder d = new Detector.Builder("count", null);
+        AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(d.build()));
+
+        Job.Builder builder = new Job.Builder();
+        builder.setId(id);
+        builder.setAnalysisConfig(analysisConfig);
+        builder.setDataDescription(dataDescription);
+        return builder;
+    }
+}
diff --git a/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java b/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java
index 2967096b00b70..924479b4f882c 100644
--- a/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.upgrades;
 
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 
 import java.util.Map;
@@ -22,7 +23,7 @@ public void testNewClusterHasActiveNonExpiringBasic() throws Exception {
     }
 
     private void checkBasicLicense() throws Exception {
-        Response licenseResponse = client().performRequest("GET", "/_xpack/license");
+        Response licenseResponse = client().performRequest(new Request("GET", "/_xpack/license"));
         Map<String, Object> licenseResponseMap = entityAsMap(licenseResponse);
         Map<String, Object> licenseMap = (Map<String, Object>) licenseResponseMap.get("license");
         assertEquals("basic", licenseMap.get("type"));
@@ -30,7 +31,7 @@ private void checkBasicLicense() throws Exception {
     }
 
     private void checkNonExpiringBasicLicense() throws Exception {
-        Response licenseResponse = client().performRequest("GET", "/_xpack/license");
+        Response licenseResponse = client().performRequest(new Request("GET", "/_xpack/license"));
         Map<String, Object> licenseResponseMap = entityAsMap(licenseResponse);
         Map<String, Object> licenseMap = (Map<String, Object>) licenseResponseMap.get("license");
         assertEquals("basic", licenseMap.get("type"));
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
index a3576b7b8c3e8..ec912a68b59c2 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.upgrades;
 
+import org.elasticsearch.client.Request;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.SecuritySettingsSourceField;
@@ -75,8 +76,9 @@ public void setupForTests() throws Exception {
             boolean success = true;
             for (String template : templatesToWaitFor()) {
                 try {
-                    final boolean exists =
-                            adminClient().performRequest("HEAD", "_template/" + template).getStatusLine().getStatusCode() == 200;
+                    final boolean exists = adminClient()
+                            .performRequest(new Request("HEAD", "_template/" + template))
+                            .getStatusLine().getStatusCode() == 200;
                     success &= exists;
                     logger.debug("template [{}] exists [{}]", template, exists);
                 } catch (IOException e) {
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java
index 92a8c3d2f830f..2a681afc14cb7 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java
@@ -5,16 +5,13 @@
  */
 package org.elasticsearch.upgrades;
 
-import org.apache.http.HttpEntity;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Booleans;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -52,7 +49,7 @@ private int expectedNumUniqueNodeNameBuckets() throws IOException {
     }
 
     private void assertAuditDocsExist() throws Exception {
-        Response response = client().performRequest("GET", "/.security_audit_log*/doc/_count");
+        Response response = client().performRequest(new Request("GET", "/.security_audit_log*/doc/_count"));
         assertEquals(200, response.getStatusLine().getStatusCode());
         Map<String, Object> responseMap = entityAsMap(response);
         assertNotNull(responseMap.get("count"));
@@ -61,19 +58,20 @@ private void assertAuditDocsExist() throws Exception {
 
     private void assertNumUniqueNodeNameBuckets(int numBuckets) throws Exception {
         // call API that will hit all nodes
-        Map<?, ?> nodesResponse = entityAsMap(client().performRequest("GET", "/_nodes/_all/info/version"));
+        Map<?, ?> nodesResponse = entityAsMap(client().performRequest(new Request("GET", "/_nodes/_all/info/version")));
         logger.info("all nodes {}", nodesResponse);
 
-        HttpEntity httpEntity = new StringEntity(
+        Request aggRequest = new Request("GET", "/.security_audit_log*/_search");
+        aggRequest.setJsonEntity(
                 "{\n" +
                 "    \"aggs\" : {\n" +
                 "        \"nodes\" : {\n" +
                 "            \"terms\" : { \"field\" : \"node_name\" }\n" +
                 "        }\n" +
                 "    }\n" +
-                "}", ContentType.APPLICATION_JSON);
-        Response aggResponse = client().performRequest("GET", "/.security_audit_log*/_search",
-                Collections.singletonMap("pretty", "true"), httpEntity);
+                "}");
+        aggRequest.addParameter("pretty", "true");
+        Response aggResponse = client().performRequest(aggRequest);
         Map<String, Object> aggResponseMap = entityAsMap(aggResponse);
         logger.debug("aggResponse {}", aggResponseMap);
         Map<?, ?> aggregations = (Map<?, ?>) aggResponseMap.get("aggregations");
@@ -89,7 +87,7 @@ private void assertNumUniqueNodeNameBuckets(int numBuckets) throws Exception {
      * Has the master been upgraded to the new version?
      */
     private boolean masterIsNewVersion() throws IOException {
-        Map<?, ?> map = entityAsMap(client().performRequest("GET", "/_nodes/_master"));
+        Map<?, ?> map = entityAsMap(client().performRequest(new Request("GET", "/_nodes/_master")));
         map = (Map<?, ?>) map.get("nodes");
         assertThat(map.values(), hasSize(1));
         map = (Map<?, ?>) map.values().iterator().next();
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RollupIDUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RollupIDUpgradeIT.java
new file mode 100644
index 0000000000000..0da79b27bfaf6
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RollupIDUpgradeIT.java
@@ -0,0 +1,299 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.hamcrest.Matcher;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+
+
+public class RollupIDUpgradeIT extends AbstractUpgradeTestCase {
+    private static final Version UPGRADE_FROM_VERSION =
+        Version.fromString(System.getProperty("tests.upgrade_from_version"));
+
+    /**
+     * This test verifies that as a cluster is upgraded incrementally, new documents eventually switch
+     * over to the "new" form of ID (128 bit Murmur3 ids).
+     *
+     * Rollup IDs are essentially the hashed concatenation of keys returned by the composite aggregation,
+     * so the field values that are being indexed (timestamp, value, etc) directly affect the
+     * ID that is generated.
+     *
+     * We don't know which node will get the Rollup task to start, so we don't know when it will migrate.
+     * The first doc is guaranteed to be the "old" style since all nodes are un-upgraded.  The second
+     * and third phase will have a mixed cluster, and the rollup task may or may not migrate.  In those
+     * phases we have two options (old and new) for the document added in the phase.
+     *
+     * The last phase is guaranteed to be new as it's a fully upgraded cluster.
+     */
+    public void testIDsUpgradeCorrectly() throws Exception {
+        assumeTrue("Rollup became available in 6.3", UPGRADE_FROM_VERSION.onOrAfter(Version.V_6_3_0));
+        assumeTrue("Rollup ID change happened in 6.4", UPGRADE_FROM_VERSION.before(Version.V_6_4_0));
+        switch (CLUSTER_TYPE) {
+            case OLD:
+                break;
+            case MIXED:
+                Request waitForYellow = new Request("GET", "/_cluster/health");
+                waitForYellow.addParameter("wait_for_nodes", "3");
+                waitForYellow.addParameter("wait_for_status", "yellow");
+                client().performRequest(waitForYellow);
+                break;
+            case UPGRADED:
+                Request waitForGreen = new Request("GET", "/_cluster/health/target,rollup");
+                waitForGreen.addParameter("wait_for_nodes", "3");
+                waitForGreen.addParameter("wait_for_status", "green");
+                // wait for long enough that we give delayed unassigned shards to stop being delayed
+                waitForGreen.addParameter("timeout", "70s");
+                waitForGreen.addParameter("level", "shards");
+                client().performRequest(waitForGreen);
+                break;
+            default:
+                throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]");
+        }
+
+        OffsetDateTime timestamp = Instant.parse("2018-01-01T00:00:01.000Z").atOffset(ZoneOffset.UTC);
+
+        if (CLUSTER_TYPE == ClusterType.OLD) {
+            String recoverQuickly = "{\"settings\": {\"index.unassigned.node_left.delayed_timeout\": \"100ms\"}}";
+
+            Request createTargetIndex = new Request("PUT", "/target");
+            createTargetIndex.setJsonEntity(recoverQuickly);
+            client().performRequest(createTargetIndex);
+
+            final Request indexRequest = new Request("POST", "/target/_doc/1");
+            indexRequest.setJsonEntity("{\"timestamp\":\"" + timestamp.toString() + "\",\"value\":123}");
+            client().performRequest(indexRequest);
+
+            // create the rollup job
+            final Request createRollupJobRequest = new Request("PUT", "/_xpack/rollup/job/rollup-id-test");
+            createRollupJobRequest.setJsonEntity("{"
+                + "\"index_pattern\":\"target\","
+                + "\"rollup_index\":\"rollup\","
+                + "\"cron\":\"*/1 * * * * ?\","
+                + "\"page_size\":100,"
+                + "\"groups\":{"
+                + "    \"date_histogram\":{"
+                + "        \"field\":\"timestamp\","
+                + "        \"interval\":\"5m\""
+                + "      },"
+                +       "\"histogram\":{"
+                + "        \"fields\": [\"value\"],"
+                + "        \"interval\":1"
+                + "      },"
+                +       "\"terms\":{"
+                + "        \"fields\": [\"value\"]"
+                + "      }"
+                + "},"
+                + "\"metrics\":["
+                + "    {\"field\":\"value\",\"metrics\":[\"min\",\"max\",\"sum\"]}"
+                + "]"
+                + "}");
+
+            Map<String, Object> createRollupJobResponse = entityAsMap(client().performRequest(createRollupJobRequest));
+            assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+            Request updateSettings = new Request("PUT", "/rollup/_settings");
+            updateSettings.setJsonEntity(recoverQuickly);
+            client().performRequest(updateSettings);
+
+            // start the rollup job
+            final Request startRollupJobRequest = new Request("POST", "_xpack/rollup/job/rollup-id-test/_start");
+            Map<String, Object> startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest));
+            assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE));
+
+            assertRollUpJob("rollup-id-test");
+            List<String> ids = getSearchResults(1);
+            assertThat(ids, containsInAnyOrder("3310683722"));
+
+        }
+
+        if (CLUSTER_TYPE == ClusterType.MIXED && Booleans.parseBoolean(System.getProperty("tests.first_round"))) {
+            final Request indexRequest = new Request("POST", "/target/_doc/2");
+            indexRequest.setJsonEntity("{\"timestamp\":\"" + timestamp.plusDays(1).toString() + "\",\"value\":345}");
+            client().performRequest(indexRequest);
+
+            assertRollUpJob("rollup-id-test");
+            client().performRequest(new Request("POST", "rollup/_refresh"));
+
+            List<String> ids = getSearchResults(2);
+            // first doc, guaranteed old style
+            ids.remove("3310683722");
+
+            // next doc may be either style
+            ids.removeAll(Arrays.asList("621059582", "rollup-id-test$ehY4NAyVSy8xxUDZrNXXIA"));
+            assertThat(ids.toString(),ids.size(), equalTo(0));
+        }
+
+        if (CLUSTER_TYPE == ClusterType.MIXED && Booleans.parseBoolean(System.getProperty("tests.first_round")) == false) {
+            final Request indexRequest = new Request("POST", "/target/_doc/3");
+            indexRequest.setJsonEntity("{\"timestamp\":\"" + timestamp.plusDays(2).toString() + "\",\"value\":456}");
+            client().performRequest(indexRequest);
+
+            assertRollUpJob("rollup-id-test");
+            client().performRequest(new Request("POST", "rollup/_refresh"));
+
+            List<String> ids = getSearchResults(3);
+
+            // first doc, guaranteed old style
+            ids.remove("3310683722");
+
+            // next two docs may be either style
+            ids.removeAll(Arrays.asList("621059582", "4288019978",
+                "rollup-id-test$ehY4NAyVSy8xxUDZrNXXIA", "rollup-id-test$60RGDSb92YI5LH4_Fnq_1g"));
+            assertThat(ids.toString(), ids.size(), equalTo(0));
+
+        }
+
+        if (CLUSTER_TYPE == ClusterType.UPGRADED) {
+            final Request indexRequest = new Request("POST", "/target/_doc/4");
+            indexRequest.setJsonEntity("{\"timestamp\":\"" + timestamp.plusDays(3).toString() + "\",\"value\":567}");
+            client().performRequest(indexRequest);
+
+            assertRollUpJob("rollup-id-test");
+            client().performRequest(new Request("POST", "rollup/_refresh"));
+
+            List<String> ids = getSearchResults(4);
+            // first doc, guaranteed old style
+            ids.remove("3310683722");
+
+            // next two docs may be either style
+            ids.removeAll(Arrays.asList("621059582", "4288019978",
+                "rollup-id-test$ehY4NAyVSy8xxUDZrNXXIA", "rollup-id-test$60RGDSb92YI5LH4_Fnq_1g"));
+
+            // Last is guaranteed to be new
+            ids.remove("rollup-id-test$LAKZftDeQwsUtdPixrkkzQ");
+            assertThat(ids.toString(), ids.size(), equalTo(0));
+        }
+
+    }
+
+    private List<String> getSearchResults(int expectedCount) throws Exception {
+        final List<String> collectedIDs = new ArrayList<>();
+        assertBusy(() -> {
+            collectedIDs.clear();
+            client().performRequest(new Request("POST", "rollup/_refresh"));
+            final Request searchRequest = new Request("GET", "rollup/_search");
+            try {
+                Map<String, Object> searchResponse = entityAsMap(client().performRequest(searchRequest));
+                assertNotNull(ObjectPath.eval("hits.total", searchResponse));
+                assertThat(ObjectPath.eval("hits.total", searchResponse), equalTo(expectedCount));
+
+                for (int i = 0; i < expectedCount; i++) {
+                    String id = ObjectPath.eval("hits.hits." + i + "._id", searchResponse);
+                    collectedIDs.add(id);
+                    Map<String, Object> doc = ObjectPath.eval("hits.hits." + i + "._source", searchResponse);
+                    assertNotNull(doc);
+                    if (id.startsWith("rollup-id-test")) {
+                        assertThat(doc.get("_rollup.version"), equalTo(2));
+                    } else {
+                        assertThat(doc.get("_rollup.version"), equalTo(1));
+                    }
+                }
+            } catch (IOException e) {
+                fail();
+            }
+        });
+        return collectedIDs;
+    }
+
+    @SuppressWarnings("unchecked")
+    private void assertRollUpJob(final String rollupJob) throws Exception {
+        final Matcher<?> expectedStates = anyOf(equalTo("indexing"), equalTo("started"));
+        waitForRollUpJob(rollupJob, expectedStates);
+
+        // check that the rollup job is started using the RollUp API
+        final Request getRollupJobRequest = new Request("GET", "_xpack/rollup/job/" + rollupJob);
+        Map<String, Object> getRollupJobResponse = entityAsMap(client().performRequest(getRollupJobRequest));
+        Map<String, Object> job = getJob(getRollupJobResponse, rollupJob);
+        if (job != null) {
+            assertThat(ObjectPath.eval("status.job_state", job), expectedStates);
+        }
+
+        // check that the rollup job is started using the Tasks API
+        final Request taskRequest = new Request("GET", "_tasks");
+        taskRequest.addParameter("detailed", "true");
+        taskRequest.addParameter("actions", "xpack/rollup/*");
+        Map<String, Object> taskResponse = entityAsMap(client().performRequest(taskRequest));
+        Map<String, Object> taskResponseNodes = (Map<String, Object>) taskResponse.get("nodes");
+        Map<String, Object> taskResponseNode = (Map<String, Object>) taskResponseNodes.values().iterator().next();
+        Map<String, Object> taskResponseTasks = (Map<String, Object>) taskResponseNode.get("tasks");
+        Map<String, Object> taskResponseStatus = (Map<String, Object>) taskResponseTasks.values().iterator().next();
+        assertThat(ObjectPath.eval("status.job_state", taskResponseStatus), expectedStates);
+
+        // check that the rollup job is started using the Cluster State API
+        final Request clusterStateRequest = new Request("GET", "_cluster/state/metadata");
+        Map<String, Object> clusterStateResponse = entityAsMap(client().performRequest(clusterStateRequest));
+        List<Map<String, Object>> rollupJobTasks = ObjectPath.eval("metadata.persistent_tasks.tasks", clusterStateResponse);
+
+        boolean hasRollupTask = false;
+        for (Map<String, Object> task : rollupJobTasks) {
+            if (ObjectPath.eval("id", task).equals(rollupJob)) {
+                hasRollupTask = true;
+                break;
+            }
+        }
+        if (hasRollupTask == false) {
+            fail("Expected persistent task for [" + rollupJob + "] but none found.");
+        }
+
+    }
+
+    private void waitForRollUpJob(final String rollupJob, final Matcher<?> expectedStates) throws Exception {
+        assertBusy(() -> {
+            final Request getRollupJobRequest = new Request("GET", "_xpack/rollup/job/" + rollupJob);
+            Response getRollupJobResponse = client().performRequest(getRollupJobRequest);
+            assertThat(getRollupJobResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+
+            Map<String, Object> job = getJob(getRollupJobResponse, rollupJob);
+            if (job != null) {
+                assertThat(ObjectPath.eval("status.job_state", job), expectedStates);
+            }
+        }, 30L, TimeUnit.SECONDS);
+    }
+
+    private static Map<String, Object> getJob(Response response, String targetJobId) throws IOException {
+        return getJob(ESRestTestCase.entityAsMap(response), targetJobId);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<String, Object> getJob(Map<String, Object> jobsMap, String targetJobId) throws IOException {
+
+        List<Map<String, Object>> jobs =
+            (List<Map<String, Object>>) XContentMapValues.extractValue("jobs", jobsMap);
+
+        if (jobs == null) {
+            return null;
+        }
+
+        for (Map<String, Object> job : jobs) {
+            String jobId = (String) ((Map<String, Object>) job.get("config")).get("id");
+            if (jobId.equals(targetJobId)) {
+                return job;
+            }
+        }
+        return null;
+    }
+}
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
index 705122252e7c3..24965efc62116 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
@@ -7,10 +7,9 @@
 
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpHost;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.message.BasicHeader;
 import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -18,7 +17,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -26,43 +24,46 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
 
     public void testGeneratingTokenInOldCluster() throws Exception {
         assumeTrue("this test should only run against the old cluster", CLUSTER_TYPE == ClusterType.OLD);
-        final StringEntity tokenPostBody = new StringEntity("{\n" +
+        Request createTokenRequest = new Request("POST", "_xpack/security/oauth2/token");
+        createTokenRequest.setJsonEntity(
+                "{\n" +
                 "    \"username\": \"test_user\",\n" +
                 "    \"password\": \"x-pack-test-password\",\n" +
                 "    \"grant_type\": \"password\"\n" +
-                "}", ContentType.APPLICATION_JSON);
-        Response response = client().performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenPostBody);
+                "}");
+        Response response = client().performRequest(createTokenRequest);
         assertOK(response);
         Map<String, Object> responseMap = entityAsMap(response);
         String token = (String) responseMap.get("access_token");
         assertNotNull(token);
         assertTokenWorks(token);
 
-        StringEntity oldClusterToken = new StringEntity("{\n" +
+        Request indexRequest1 = new Request("PUT", "token_backwards_compatibility_it/doc/old_cluster_token1");
+        indexRequest1.setJsonEntity(
+                "{\n" +
                 "    \"token\": \"" + token + "\"\n" +
-                "}", ContentType.APPLICATION_JSON);
-        Response indexResponse = client().performRequest("PUT", "token_backwards_compatibility_it/doc/old_cluster_token1",
-                Collections.emptyMap(), oldClusterToken);
-        assertOK(indexResponse);
+                "}");
+        client().performRequest(indexRequest1);
 
-        response = client().performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenPostBody);
-        assertOK(response);
+        Request createSecondTokenRequest = new Request("POST", "_xpack/security/oauth2/token");
+        createSecondTokenRequest.setEntity(createTokenRequest.getEntity());
+        response = client().performRequest(createSecondTokenRequest);
         responseMap = entityAsMap(response);
         token = (String) responseMap.get("access_token");
         assertNotNull(token);
         assertTokenWorks(token);
-        oldClusterToken = new StringEntity("{\n" +
+        Request indexRequest2 = new Request("PUT", "token_backwards_compatibility_it/doc/old_cluster_token2");
+        indexRequest2.setJsonEntity(
+                "{\n" +
                 "    \"token\": \"" + token + "\"\n" +
-                "}", ContentType.APPLICATION_JSON);
-        indexResponse = client().performRequest("PUT", "token_backwards_compatibility_it/doc/old_cluster_token2",
-                Collections.emptyMap(), oldClusterToken);
-        assertOK(indexResponse);
+                "}");
+        client().performRequest(indexRequest2);
     }
 
     public void testTokenWorksInMixedOrUpgradedCluster() throws Exception {
         assumeTrue("this test should only run against the mixed or upgraded cluster",
                 CLUSTER_TYPE == ClusterType.MIXED || CLUSTER_TYPE == ClusterType.UPGRADED);
-        Response getResponse = client().performRequest("GET", "token_backwards_compatibility_it/doc/old_cluster_token1");
+        Response getResponse = client().performRequest(new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token1"));
         assertOK(getResponse);
         Map<String, Object> source = (Map<String, Object>) entityAsMap(getResponse).get("_source");
         assertTokenWorks((String) source.get("token"));
@@ -71,26 +72,27 @@ public void testTokenWorksInMixedOrUpgradedCluster() throws Exception {
     public void testMixedCluster() throws Exception {
         assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED);
         assumeTrue("the master must be on the latest version before we can write", isMasterOnLatestVersion());
-        Response getResponse = client().performRequest("GET", "token_backwards_compatibility_it/doc/old_cluster_token2");
-        assertOK(getResponse);
+        Response getResponse = client().performRequest(new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token2"));
         Map<String, Object> source = (Map<String, Object>) entityAsMap(getResponse).get("_source");
         final String token = (String) source.get("token");
         assertTokenWorks(token);
 
-        final StringEntity body = new StringEntity("{\"token\": \"" + token + "\"}", ContentType.APPLICATION_JSON);
-        Response invalidationResponse = client().performRequest("DELETE", "_xpack/security/oauth2/token", Collections.emptyMap(), body);
-        assertOK(invalidationResponse);
+        Request invalidateRequest = new Request("DELETE", "_xpack/security/oauth2/token");
+        invalidateRequest.setJsonEntity("{\"token\": \"" + token + "\"}");
+        invalidateRequest.addParameter("error_trace", "true");
+        client().performRequest(invalidateRequest);
         assertTokenDoesNotWork(token);
 
         // create token and refresh on version that supports it
-        final StringEntity tokenPostBody = new StringEntity("{\n" +
+        Request createTokenRequest = new Request("POST", "_xpack/security/oauth2/token");
+        createTokenRequest.setJsonEntity(
+                "{\n" +
                 "    \"username\": \"test_user\",\n" +
                 "    \"password\": \"x-pack-test-password\",\n" +
                 "    \"grant_type\": \"password\"\n" +
-                "}", ContentType.APPLICATION_JSON);
+                "}");
         try (RestClient client = getRestClientForCurrentVersionNodesOnly()) {
-            Response response = client.performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenPostBody);
-            assertOK(response);
+            Response response = client.performRequest(createTokenRequest);
             Map<String, Object> responseMap = entityAsMap(response);
             String accessToken = (String) responseMap.get("access_token");
             String refreshToken = (String) responseMap.get("refresh_token");
@@ -98,12 +100,13 @@ public void testMixedCluster() throws Exception {
             assertNotNull(refreshToken);
             assertTokenWorks(accessToken);
 
-            final StringEntity tokenRefresh = new StringEntity("{\n" +
+            Request tokenRefreshRequest = new Request("POST", "_xpack/security/oauth2/token");
+            tokenRefreshRequest.setJsonEntity(
+                    "{\n" +
                     "    \"refresh_token\": \"" + refreshToken + "\",\n" +
                     "    \"grant_type\": \"refresh_token\"\n" +
-                    "}", ContentType.APPLICATION_JSON);
-            response = client.performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenRefresh);
-            assertOK(response);
+                    "}");
+            response = client.performRequest(tokenRefreshRequest);
             responseMap = entityAsMap(response);
             String updatedAccessToken = (String) responseMap.get("access_token");
             String updatedRefreshToken = (String) responseMap.get("refresh_token");
@@ -118,31 +121,32 @@ public void testMixedCluster() throws Exception {
 
     public void testUpgradedCluster() throws Exception {
         assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.UPGRADED);
-        Response getResponse = client().performRequest("GET", "token_backwards_compatibility_it/doc/old_cluster_token2");
+        Response getResponse = client().performRequest(new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token2"));
         assertOK(getResponse);
         Map<String, Object> source = (Map<String, Object>) entityAsMap(getResponse).get("_source");
         final String token = (String) source.get("token");
 
         // invalidate again since this may not have been invalidated in the mixed cluster
-        final StringEntity body = new StringEntity("{\"token\": \"" + token + "\"}", ContentType.APPLICATION_JSON);
-        Response invalidationResponse = client().performRequest("DELETE", "_xpack/security/oauth2/token",
-                Collections.singletonMap("error_trace", "true"), body);
+        Request invalidateRequest = new Request("DELETE", "_xpack/security/oauth2/token");
+        invalidateRequest.setJsonEntity("{\"token\": \"" + token + "\"}");
+        invalidateRequest.addParameter("error_trace", "true");
+        Response invalidationResponse = client().performRequest(invalidateRequest);
         assertOK(invalidationResponse);
         assertTokenDoesNotWork(token);
 
-        getResponse = client().performRequest("GET", "token_backwards_compatibility_it/doc/old_cluster_token1");
-        assertOK(getResponse);
+        getResponse = client().performRequest(new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token1"));
         source = (Map<String, Object>) entityAsMap(getResponse).get("_source");
         final String workingToken = (String) source.get("token");
         assertTokenWorks(workingToken);
 
-        final StringEntity tokenPostBody = new StringEntity("{\n" +
+        Request getTokenRequest = new Request("POST", "_xpack/security/oauth2/token");
+        getTokenRequest.setJsonEntity(
+                "{\n" +
                 "    \"username\": \"test_user\",\n" +
                 "    \"password\": \"x-pack-test-password\",\n" +
                 "    \"grant_type\": \"password\"\n" +
-                "}", ContentType.APPLICATION_JSON);
-        Response response = client().performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenPostBody);
-        assertOK(response);
+                "}");
+        Response response = client().performRequest(getTokenRequest);
         Map<String, Object> responseMap = entityAsMap(response);
         String accessToken = (String) responseMap.get("access_token");
         String refreshToken = (String) responseMap.get("refresh_token");
@@ -150,12 +154,13 @@ public void testUpgradedCluster() throws Exception {
         assertNotNull(refreshToken);
         assertTokenWorks(accessToken);
 
-        final StringEntity tokenRefresh = new StringEntity("{\n" +
+        Request refreshTokenRequest = new Request("POST", "_xpack/security/oauth2/token");
+        refreshTokenRequest.setJsonEntity(
+                "{\n" +
                 "    \"refresh_token\": \"" + refreshToken + "\",\n" +
                 "    \"grant_type\": \"refresh_token\"\n" +
-                "}", ContentType.APPLICATION_JSON);
-        response = client().performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenRefresh);
-        assertOK(response);
+                "}");
+        response = client().performRequest(refreshTokenRequest);
         responseMap = entityAsMap(response);
         String updatedAccessToken = (String) responseMap.get("access_token");
         String updatedRefreshToken = (String) responseMap.get("refresh_token");
@@ -168,16 +173,21 @@ public void testUpgradedCluster() throws Exception {
     }
 
     private void assertTokenWorks(String token) throws IOException {
-        Response authenticateResponse = client().performRequest("GET", "_xpack/security/_authenticate", Collections.emptyMap(),
-                new BasicHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token));
+        Request request = new Request("GET", "_xpack/security/_authenticate");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token);
+        request.setOptions(options);
+        Response authenticateResponse = client().performRequest(request);
         assertOK(authenticateResponse);
         assertEquals("test_user", entityAsMap(authenticateResponse).get("username"));
     }
 
     private void assertTokenDoesNotWork(String token) {
-        ResponseException e = expectThrows(ResponseException.class,
-                () -> client().performRequest("GET", "_xpack/security/_authenticate", Collections.emptyMap(),
-                        new BasicHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token)));
+        Request request = new Request("GET", "_xpack/security/_authenticate");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token);
+        request.setOptions(options);
+        ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
         assertEquals(401, e.getResponse().getStatusLine().getStatusCode());
         Response response = e.getResponse();
         assertEquals("Bearer realm=\"security\", error=\"invalid_token\", error_description=\"The access token expired\"",
@@ -185,17 +195,17 @@ private void assertTokenDoesNotWork(String token) {
     }
 
     private boolean isMasterOnLatestVersion() throws Exception {
-        Response response = client().performRequest("GET", "_cluster/state");
+        Response response = client().performRequest(new Request("GET", "_cluster/state"));
         assertOK(response);
         final String masterNodeId = ObjectPath.createFromResponse(response).evaluate("master_node");
-        response = client().performRequest("GET", "_nodes");
+        response = client().performRequest(new Request("GET", "_nodes"));
         assertOK(response);
         ObjectPath objectPath = ObjectPath.createFromResponse(response);
         return Version.CURRENT.equals(Version.fromString(objectPath.evaluate("nodes." + masterNodeId + ".version")));
     }
 
     private RestClient getRestClientForCurrentVersionNodesOnly() throws IOException {
-        Response response = client().performRequest("GET", "_nodes");
+        Response response = client().performRequest(new Request("GET", "_nodes"));
         assertOK(response);
         ObjectPath objectPath = ObjectPath.createFromResponse(response);
         Map<String, Object> nodesAsMap = objectPath.evaluate("nodes");
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java
new file mode 100644
index 0000000000000..395ab375bd249
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.upgrades;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+
+public class WatcherRestartIT extends AbstractUpgradeTestCase {
+
+    public void testWatcherRestart() throws Exception {
+        client().performRequest(new Request("POST", "/_xpack/watcher/_stop"));
+        ensureWatcherStopped();
+
+        client().performRequest(new Request("POST", "/_xpack/watcher/_start"));
+        ensureWatcherStarted();
+    }
+
+    private void ensureWatcherStopped() throws Exception {
+        assertBusy(() -> {
+            Response stats = client().performRequest(new Request("GET", "_xpack/watcher/stats"));
+            String responseBody = EntityUtils.toString(stats.getEntity(), StandardCharsets.UTF_8);
+            assertThat(responseBody, containsString("\"watcher_state\":\"stopped\""));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"starting\"")));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"started\"")));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"stopping\"")));
+        });
+    }
+
+    private void ensureWatcherStarted() throws Exception {
+        assertBusy(() -> {
+            Response response = client().performRequest(new Request("GET", "_xpack/watcher/stats"));
+            String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+            assertThat(responseBody, containsString("\"watcher_state\":\"started\""));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"starting\"")));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"stopping\"")));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"stopped\"")));
+        });
+    }
+}
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_watcher.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_watcher.yml
new file mode 100644
index 0000000000000..30a22e636732f
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_watcher.yml
@@ -0,0 +1,74 @@
+---
+"CRUD watch APIs":
+
+  # no need to put watch, exists already
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+
+
+  # execute watch
+  - do:
+      xpack.watcher.execute_watch:
+        id: "my_watch"
+        body:  >
+          {
+            "record_execution" : true
+          }
+  - set: { _id : record_id }
+  - match: { watch_record.watch_id: "my_watch" }
+  - match: { watch_record.trigger_event.type: "manual" }
+  - match: { watch_record.state: "executed" }
+  - match: { watch_record.status.execution_state: "executed" }
+  - match: { watch_record.status.state.active: true }
+
+  # check watch history entry
+  - do:
+      indices.refresh:
+        index: .watcher-history-*
+
+  - do:
+      search:
+        index: .watcher-history-*
+        body:
+          {
+            "query" : { "term" : { "_id" : "$record_id" } }
+          }
+  - match: { hits.total: 1 }
+
+  # deactivate watch, check with GET API as well
+  - do:
+      xpack.watcher.deactivate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : false }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: false }
+
+
+  # activate watch again, check with GET API as well
+  - do:
+      xpack.watcher.activate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : true }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: true }
+
+
+---
+"Test watcher stats output":
+  - do:
+      xpack.watcher.stats: {}
+  - match: { "manually_stopped": false }
+  - match: { "stats.0.watcher_state": "started" }
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/60_watcher.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/60_watcher.yml
new file mode 100644
index 0000000000000..2547ddd7023df
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/60_watcher.yml
@@ -0,0 +1,94 @@
+---
+"CRUD watch APIs":
+
+  - do:
+      xpack.watcher.put_watch:
+        id: "my_watch"
+        body:  >
+          {
+            "trigger": {
+              "schedule" : { "cron" : "0 0 0 1 * ? 2099" }
+            },
+            "input": {
+              "simple": {}
+            },
+            "actions": {
+              "logging": {
+                "logging": {
+                  "text": "my logging action"
+                }
+              }
+            }
+          }
+  - match: { _id: "my_watch" }
+
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+
+
+  # execute watch
+  - do:
+      xpack.watcher.execute_watch:
+        id: "my_watch"
+        body:  >
+          {
+            "record_execution" : true
+          }
+  - set: { _id : record_id }
+  - match: { watch_record.watch_id: "my_watch" }
+  - match: { watch_record.trigger_event.type: "manual" }
+  - match: { watch_record.state: "executed" }
+  - match: { watch_record.status.execution_state: "executed" }
+  - match: { watch_record.status.state.active: true }
+
+  # check watch history entry
+  - do:
+      indices.refresh:
+        index: .watcher-history-*
+
+  - do:
+      search:
+        index: .watcher-history-*
+        body:
+          {
+            "query" : { "term" : { "_id" : "$record_id" } }
+          }
+  - match: { hits.total: 1 }
+
+  # deactivate watch, check with GET API as well
+  - do:
+      xpack.watcher.deactivate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : false }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: false }
+
+
+  # activate watch again, check with GET API as well
+  - do:
+      xpack.watcher.activate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : true }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: true }
+
+
+---
+"Test watcher stats output":
+  - do:
+      xpack.watcher.stats: {}
+  - match: { "manually_stopped": false }
+  - match: { "stats.0.watcher_state": "started" }
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_watcher.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_watcher.yml
new file mode 100644
index 0000000000000..75e31c62a840e
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_watcher.yml
@@ -0,0 +1,73 @@
+---
+"CRUD watch APIs":
+
+  # no need to put watch, exists already
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+
+
+  # execute watch
+  - do:
+      xpack.watcher.execute_watch:
+        id: "my_watch"
+        body:  >
+          {
+            "record_execution" : true
+          }
+  - set: { _id : record_id }
+  - match: { watch_record.watch_id: "my_watch" }
+  - match: { watch_record.trigger_event.type: "manual" }
+  - match: { watch_record.state: "executed" }
+  - match: { watch_record.status.execution_state: "executed" }
+  - match: { watch_record.status.state.active: true }
+
+  # check watch history entry
+  - do:
+      indices.refresh:
+        index: .watcher-history-*
+
+  - do:
+      search:
+        index: .watcher-history-*
+        body:
+          {
+            "query" : { "term" : { "_id" : "$record_id" } }
+          }
+  - match: { hits.total: 1 }
+
+  # deactivate watch, check with GET API as well
+  - do:
+      xpack.watcher.deactivate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : false }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: false }
+
+
+  # activate watch again, check with GET API as well
+  - do:
+      xpack.watcher.activate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : true }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: true }
+
+---
+"Test watcher stats output":
+  - do:
+      xpack.watcher.stats: {}
+  - match: { "manually_stopped": false }
+  - match: { "stats.0.watcher_state": "started" }
diff --git a/x-pack/qa/rolling-upgrade/with-system-key/build.gradle b/x-pack/qa/rolling-upgrade/with-system-key/build.gradle
index 03505e01dedd8..5aaa1ed1eff9b 100644
--- a/x-pack/qa/rolling-upgrade/with-system-key/build.gradle
+++ b/x-pack/qa/rolling-upgrade/with-system-key/build.gradle
@@ -1 +1,10 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+// Skip test on FIPS FIXME https://github.com/elastic/elasticsearch/issues/32737
+if (project.inFipsJvm) {
+    tasks.withType(RestIntegTestTask) {
+        enabled = false
+    }
+}
+
 group = "${group}.x-pack.qa.rolling-upgrade.with-system-key"
diff --git a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java
index 19ef9d2eb0d72..af3fb160e133f 100644
--- a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java
+++ b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 public class CustomRealm extends Realm {
 
diff --git a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java
index 65ec595a0d409..f7bdb0d0baa9a 100644
--- a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java
+++ b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java
@@ -5,10 +5,11 @@
  */
 package org.elasticsearch.example.realm;
 
-import org.apache.http.message.BasicHeader;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
@@ -50,7 +51,7 @@ protected Collection<Class<? extends Plugin>> transportClientPlugins() {
 
     public void testHttpConnectionWithNoAuthentication() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/");
+            getRestClient().performRequest(new Request("GET", "/"));
             fail("request should have failed");
         } catch(ResponseException e) {
             Response response = e.getResponse();
@@ -61,9 +62,12 @@ public void testHttpConnectionWithNoAuthentication() throws Exception {
     }
 
     public void testHttpAuthentication() throws Exception {
-        Response response = getRestClient().performRequest("GET", "/",
-                new BasicHeader(CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER),
-                new BasicHeader(CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString()));
+        Request request = new Request("GET", "/");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader(CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER);
+        options.addHeader(CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString());
+        request.setOptions(options);
+        Response response = getRestClient().performRequest(request);
         assertThat(response.getStatusLine().getStatusCode(), is(200));
     }
 
diff --git a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java
index d1435ebaa3c28..e206de6e39272 100644
--- a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java
+++ b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
diff --git a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java
index bd3c53a3b41be..57a895848e3a8 100644
--- a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java
+++ b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java
@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.example.role;
 
-import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.network.NetworkModule;
@@ -33,10 +34,17 @@
  * Integration test for custom roles providers.
  */
 public class CustomRolesProviderIT extends ESIntegTestCase {
-
     private static final String TEST_USER = "test_user";
     private static final String TEST_PWD = "change_me";
 
+    private static final RequestOptions AUTH_OPTIONS;
+    static {
+        RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+        options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+                basicAuthHeaderValue(TEST_USER, new SecureString(TEST_PWD.toCharArray())));
+        AUTH_OPTIONS = options.build();
+    }
+
     @Override
     protected Settings externalClusterClientSettings() {
         return Settings.builder()
@@ -59,7 +67,9 @@ public void setupTestUser(String role) {
     public void testAuthorizedCustomRoleSucceeds() throws Exception {
         setupTestUser(ROLE_B);
         // roleB has all permissions on index "foo", so creating "foo" should succeed
-        Response response = getRestClient().performRequest("PUT", "/" + INDEX, authHeader());
+        Request request = new Request("PUT", "/" + INDEX);
+        request.setOptions(AUTH_OPTIONS);
+        Response response = getRestClient().performRequest(request);
         assertThat(response.getStatusLine().getStatusCode(), is(200));
     }
 
@@ -71,7 +81,9 @@ public void testFirstResolvedRoleTakesPrecedence() throws Exception {
         setupTestUser(ROLE_A);
         // roleB has all permissions on index "foo", so creating "foo" should succeed
         try {
-            getRestClient().performRequest("PUT", "/" + INDEX, authHeader());
+            Request request = new Request("PUT", "/" + INDEX);
+            request.setOptions(AUTH_OPTIONS);
+            getRestClient().performRequest(request);
             fail(ROLE_A + " should not be authorized to create index " + INDEX);
         } catch (ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
@@ -82,15 +94,12 @@ public void testUnresolvedRoleDoesntSucceed() throws Exception {
         setupTestUser("unknown");
         // roleB has all permissions on index "foo", so creating "foo" should succeed
         try {
-            getRestClient().performRequest("PUT", "/" + INDEX, authHeader());
+            Request request = new Request("PUT", "/" + INDEX);
+            request.setOptions(AUTH_OPTIONS);
+            getRestClient().performRequest(request);
             fail(ROLE_A + " should not be authorized to create index " + INDEX);
         } catch (ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
         }
     }
-
-    private BasicHeader authHeader() {
-        return new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                               basicAuthHeaderValue(TEST_USER, new SecureString(TEST_PWD.toCharArray())));
-    }
 }
diff --git a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java b/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java
index 4ac927c6646c1..e810e638f6895 100644
--- a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java
+++ b/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
 import org.elasticsearch.xpack.core.security.client.SecurityClient;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.protocol.xpack.security.User;
 import org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool;
 import org.junit.Before;
 
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShardFailureIT.java b/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShardFailureIT.java
new file mode 100644
index 0000000000000..e88b6b94d209f
--- /dev/null
+++ b/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShardFailureIT.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.qa.sql.nosecurity;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.xpack.qa.sql.jdbc.JdbcIntegrationTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class JdbcShardFailureIT extends JdbcIntegrationTestCase {
+    @Before
+    public void createTestIndex() throws IOException {
+        Request createTest1 = new Request("PUT", "/test1");
+        String body1 = "{\"aliases\":{\"test\":{}}, \"mappings\": {\"doc\": {\"properties\": {\"test_field\":{\"type\":\"integer\"}}}}}";
+        createTest1.setJsonEntity(body1);
+        client().performRequest(createTest1);
+
+        Request createTest2 = new Request("PUT", "/test2");
+        String body2 = "{\"aliases\":{\"test\":{}}, \"mappings\": {\"doc\": {\"properties\": {\"test_field\":{\"type\":\"integer\"}}}}," +
+            "\"settings\": {\"index.routing.allocation.include.node\": \"nowhere\"}}";
+        createTest2.setJsonEntity(body2);
+        createTest2.addParameter("timeout", "100ms");
+        client().performRequest(createTest2);
+
+        Request request = new Request("PUT", "/test1/doc/_bulk");
+        request.addParameter("refresh", "true");
+        StringBuilder bulk = new StringBuilder();
+        for (int i = 0; i < 20; i++) {
+            bulk.append("{\"index\":{}}\n");
+            bulk.append("{\"test_field\":" + i + "}\n");
+        }
+        request.setJsonEntity(bulk.toString());
+        client().performRequest(request);
+    }
+
+    public void testPartialResponseHandling() throws SQLException {
+        try (Connection c = esJdbc(); Statement s = c.createStatement()) {
+            SQLException exception = expectThrows(SQLException.class, () -> s.executeQuery("SELECT * FROM test ORDER BY test_field ASC"));
+            assertThat(exception.getMessage(), containsString("Search rejected due to missing shards"));
+        }
+    }
+}
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java
index 32d1a67e5620f..f5b9381c54b31 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java
+++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java
@@ -1,5 +1,5 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
@@ -51,6 +51,8 @@ public void testShowFunctionsLikePrefix() throws IOException {
         assertThat(readLine(), RegexMatcher.matches("\\s*LCASE\\s*\\|\\s*SCALAR\\s*"));
         assertThat(readLine(), RegexMatcher.matches("\\s*LENGTH\\s*\\|\\s*SCALAR\\s*"));
         assertThat(readLine(), RegexMatcher.matches("\\s*LTRIM\\s*\\|\\s*SCALAR\\s*"));
+        assertThat(readLine(), RegexMatcher.matches("\\s*LEFT\\s*\\|\\s*SCALAR\\s*"));
+        assertThat(readLine(), RegexMatcher.matches("\\s*LOCATE\\s*\\|\\s*SCALAR\\s*"));
         assertEquals("", readLine());
     }
 
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvTestUtils.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvTestUtils.java
index ad26db3104758..abf84dc7311d8 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvTestUtils.java
+++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvTestUtils.java
@@ -145,6 +145,8 @@ private static String resolveColumnType(String type) {
                 return "double";
             case "ts":
                 return "timestamp";
+            case "bt":
+                return "byte";
             default:
                 return type;
         }
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java
index b782e1474ea85..d61b4b9a946bd 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java
+++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java
@@ -7,12 +7,14 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.junit.Assume;
 import org.junit.ClassRule;
 
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 
 /**
  * Tests comparing sql queries executed against our jdbc client
@@ -25,7 +27,7 @@ public abstract class SqlSpecTestCase extends SpecBaseIntegrationTestCase {
     public static LocalH2 H2 = new LocalH2((c) -> c.createStatement().execute("RUNSCRIPT FROM 'classpath:/setup_test_emp.sql'"));
 
     @ParametersFactory(argumentFormatting = PARAM_FORMATTING)
-    public static List<Object[]> readScriptSpec() throws Exception {
+    public static List<Object[]> readScriptSpec() throws Exception { 
         Parser parser = specParser();
         List<Object[]> tests = new ArrayList<>();
         tests.addAll(readScriptSpec("/select.sql-spec", parser));
@@ -35,6 +37,8 @@ public static List<Object[]> readScriptSpec() throws Exception {
         tests.addAll(readScriptSpec("/agg.sql-spec", parser));
         tests.addAll(readScriptSpec("/arithmetic.sql-spec", parser));
         tests.addAll(readScriptSpec("/string-functions.sql-spec", parser));
+        // AwaitsFix: https://github.com/elastic/elasticsearch/issues/32589
+        // tests.addAll(readScriptSpec("/case-functions.sql-spec", parser));
         return tests;
     }
 
@@ -56,6 +60,12 @@ public SqlSpecTestCase(String fileName, String groupName, String testName, Integ
 
     @Override
     protected final void doTest() throws Throwable {
+        boolean goodLocale = !(Locale.getDefault().equals(new Locale.Builder().setLanguageTag("tr").build())
+                || Locale.getDefault().equals(new Locale.Builder().setLanguageTag("tr-TR").build()));
+        if (fileName.startsWith("case-functions")) {
+            Assume.assumeTrue(goodLocale);
+        }
+        
         try (Connection h2 = H2.get();
              Connection es = esJdbc()) {
 
diff --git a/x-pack/qa/sql/src/main/resources/case-functions.sql-spec b/x-pack/qa/sql/src/main/resources/case-functions.sql-spec
new file mode 100644
index 0000000000000..899d7cb0a6cb1
--- /dev/null
+++ b/x-pack/qa/sql/src/main/resources/case-functions.sql-spec
@@ -0,0 +1,13 @@
+// Next 4 SELECTs in this file are related to https://github.com/elastic/elasticsearch/issues/32589
+// H2 is Locale sensitive, while ES-SQL is not (so far)
+selectInsertWithLcaseAndLengthWithOrderBy
+SELECT "first_name" origFN, "last_name" origLN, INSERT(UCASE("first_name"),LENGTH("first_name")+1,123,LCASE("last_name")) modified FROM "test_emp" WHERE ASCII("first_name")=65 ORDER BY "first_name" ASC, "last_name" ASC LIMIT 10;
+
+upperCasingTheSecondLetterFromTheRightFromFirstName
+SELECT CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) f FROM "test_emp" ORDER BY "first_name" LIMIT 10;
+
+upperCasingTheSecondLetterFromTheRightFromFirstNameWithOrderByAndGroupBy
+SELECT CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) f, COUNT(*) c FROM "test_emp" GROUP BY CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) ORDER BY CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) LIMIT 10;
+
+upperCasingTheSecondLetterFromTheRightFromFirstNameWithWhere
+SELECT CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) f, COUNT(*) c FROM "test_emp" WHERE CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1))='AlejandRo' GROUP BY CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) ORDER BY CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) LIMIT 10;
diff --git a/x-pack/qa/sql/src/main/resources/command.csv-spec b/x-pack/qa/sql/src/main/resources/command.csv-spec
index 47615a4c3ae84..89e86e887e140 100644
--- a/x-pack/qa/sql/src/main/resources/command.csv-spec
+++ b/x-pack/qa/sql/src/main/resources/command.csv-spec
@@ -78,6 +78,15 @@ LENGTH          |SCALAR
 LTRIM           |SCALAR
 RTRIM           |SCALAR
 SPACE           |SCALAR
+CONCAT          |SCALAR
+INSERT          |SCALAR
+LEFT            |SCALAR
+LOCATE          |SCALAR
+POSITION        |SCALAR
+REPEAT          |SCALAR
+REPLACE         |SCALAR
+RIGHT           |SCALAR
+SUBSTRING       |SCALAR
 UCASE           |SCALAR
 SCORE           |SCORE
 ;
diff --git a/x-pack/qa/sql/src/main/resources/docs.csv-spec b/x-pack/qa/sql/src/main/resources/docs.csv-spec
index 8d7debee331fc..7385bf14df33b 100644
--- a/x-pack/qa/sql/src/main/resources/docs.csv-spec
+++ b/x-pack/qa/sql/src/main/resources/docs.csv-spec
@@ -232,6 +232,15 @@ LTRIM           |SCALAR
 RTRIM           |SCALAR         
 SPACE           |SCALAR         
 UCASE           |SCALAR         
+CONCAT          |SCALAR         
+INSERT          |SCALAR         
+LEFT            |SCALAR         
+LOCATE          |SCALAR         
+POSITION        |SCALAR         
+REPEAT          |SCALAR         
+REPLACE         |SCALAR         
+RIGHT           |SCALAR         
+SUBSTRING       |SCALAR         
 SCORE           |SCORE   
 
 // end::showFunctions
diff --git a/x-pack/qa/sql/src/main/resources/example.csv-spec b/x-pack/qa/sql/src/main/resources/example.csv-spec
index a7964cec8d238..77e9947af9369 100644
--- a/x-pack/qa/sql/src/main/resources/example.csv-spec
+++ b/x-pack/qa/sql/src/main/resources/example.csv-spec
@@ -12,7 +12,7 @@ SELECT COUNT(*) FROM "emp";
 
 // list of <ColumnName:ColumnType*>
 // type might be missing in which case it will be autodetected or can be one of the following
-// d - double, f - float, i - int, b - byte, l - long, t - timestamp, date
+// d - double, f - float, i - int, bt - byte, b - boolean, l - long, t - timestamp, date
 A,B:d,C:i
 // actual values
 foo,2.5,3
diff --git a/x-pack/qa/sql/src/main/resources/functions.csv-spec b/x-pack/qa/sql/src/main/resources/functions.csv-spec
index 09320c3d384c0..1a610aec04861 100644
--- a/x-pack/qa/sql/src/main/resources/functions.csv-spec
+++ b/x-pack/qa/sql/src/main/resources/functions.csv-spec
@@ -28,3 +28,382 @@ len:i          |  first_name:s
 72             |Prasadram
 88             |Sreekrishna
 ;
+
+selectConcatWithOrderBy
+SELECT first_name f, last_name l, CONCAT(first_name,last_name) cct FROM test_emp ORDER BY CONCAT(first_name,last_name) LIMIT 10;
+
+f:s        | l:s          | cct:s
+Adamantios | Portugali    | AdamantiosPortugali
+Alain      | Chappelet    | AlainChappelet
+Alejandro  | Brender      | AlejandroBrender
+Alejandro  | McAlpine     | AlejandroMcAlpine
+Amabile    | Gomatam      | AmabileGomatam
+Anneke     | Preusig      | AnnekePreusig
+Anoosh     | Peyn         | AnooshPeyn
+Arif       | Merlo        | ArifMerlo
+Arumugam   | Ossenbruggen | ArumugamOssenbruggen
+Bader      | Swan         | BaderSwan
+;
+
+selectNestedConcatWithOrderBy
+SELECT first_name f, last_name l, CONCAT(first_name,CONCAT(' ',last_name)) cct FROM test_emp ORDER BY CONCAT(first_name,CONCAT(' ',last_name)) LIMIT 10;
+
+f:s        | l:s          | cct:s
+Adamantios | Portugali    | Adamantios Portugali
+Alain      | Chappelet    | Alain Chappelet
+Alejandro  | Brender      | Alejandro Brender
+Alejandro  | McAlpine     | Alejandro McAlpine
+Amabile    | Gomatam      | Amabile Gomatam
+Anneke     | Preusig      | Anneke Preusig
+Anoosh     | Peyn         | Anoosh Peyn
+Arif       | Merlo        | Arif Merlo
+Arumugam   | Ossenbruggen | Arumugam Ossenbruggen
+Bader      | Swan         | Bader Swan
+;
+
+selectConcatWithGroupBy
+SELECT CONCAT(first_name,last_name) cct FROM test_emp GROUP BY CONCAT(first_name,last_name) ORDER BY CONCAT(first_name,last_name) LIMIT 1;
+
+cct:s
+AdamantiosPortugali
+;
+
+selectAsciiOfConcatWithGroupByOrderByCount
+SELECT ASCII(CONCAT("first_name","last_name")) ascii, COUNT(*) count FROM "test_emp" GROUP BY ASCII(CONCAT("first_name","last_name")) ORDER BY ASCII(CONCAT("first_name","last_name")) DESC LIMIT 10;
+
+ascii:i  | count:l  
+90       | 2
+89       | 3
+88       | 1
+87       | 1
+86       | 3
+85       | 2
+84       | 3
+83       | 11
+82       | 3
+80       | 6
+;
+
+selectRepeatTwice
+SELECT "first_name" orig, REPEAT("first_name",2) reps FROM "test_emp" WHERE ASCII("first_name")=65 ORDER BY REPEAT("first_name",2) ASC LIMIT 10;
+
+orig:s     | reps:s  
+Adamantios | AdamantiosAdamantios
+Alain      | AlainAlain
+Alejandro  | AlejandroAlejandro
+Alejandro  | AlejandroAlejandro
+Amabile    | AmabileAmabile
+Anneke     | AnnekeAnneke
+Anoosh     | AnooshAnoosh
+Arif       | ArifArif
+Arumugam   | ArumugamArumugam
+;
+
+selectInsertWithLcase
+SELECT "first_name" orig, INSERT("first_name",2,1000,LCASE("first_name")) modified FROM "test_emp" WHERE ASCII("first_name")=65 ORDER BY "first_name" ASC LIMIT 10;
+
+orig:s     | modified:s  
+Adamantios | Aadamantios
+Alain      | Aalain
+Alejandro  | Aalejandro
+Alejandro  | Aalejandro
+Amabile    | Aamabile
+Anneke     | Aanneke
+Anoosh     | Aanoosh
+Arif       | Aarif
+Arumugam   | Aarumugam
+;
+
+selectInsertWithLcaseAndLengthWithOrderBy
+SELECT "first_name" origFN, "last_name" origLN, INSERT(UCASE("first_name"),LENGTH("first_name")+1,123,LCASE("last_name")) modified FROM "test_emp" WHERE ASCII("first_name")=65 ORDER BY "first_name" ASC LIMIT 10;
+
+origFN:s   | origLN:s     | modified:s  
+Adamantios | Portugali    | ADAMANTIOSportugali
+Alain      | Chappelet    | ALAINchappelet
+Alejandro  | Brender      | ALEJANDRObrender
+Alejandro  | McAlpine     | ALEJANDROmcalpine
+Amabile    | Gomatam      | AMABILEgomatam
+Anneke     | Preusig      | ANNEKEpreusig
+Anoosh     | Peyn         | ANOOSHpeyn
+Arif       | Merlo        | ARIFmerlo
+Arumugam   | Ossenbruggen | ARUMUGAMossenbruggen
+;
+
+selectInsertWithUcaseWithGroupByAndOrderBy
+SELECT INSERT(UCASE("first_name"),2,123000,INSERT(UCASE("last_name"),2,500,'')) modified, COUNT(*) count FROM "test_emp" WHERE ASCII("first_name")=65 GROUP BY INSERT(UCASE("first_name"),2,123000,INSERT(UCASE("last_name"),2,500,'')) ORDER BY INSERT(UCASE("first_name"),2,123000,INSERT(UCASE("last_name"),2,500,'')) ASC LIMIT 10;
+
+modified:s  | count:l
+AB          | 1
+AC          | 1
+AG          | 1
+AM          | 2
+AO          | 1
+AP          | 3
+;
+
+selectSubstringWithGroupBy
+SELECT SUBSTRING("first_name",1,2) modified, COUNT(*) count FROM "test_emp" WHERE ASCII("first_name")=65 GROUP BY SUBSTRING("first_name",1,2) ORDER BY SUBSTRING("first_name",1,2) ASC LIMIT 10;
+
+modified:s  | count:l
+Ad          | 1
+Al          | 3
+Am          | 1
+An          | 2
+Ar          | 2
+;
+
+selectSubstringWithWhereCountAndGroupBy
+SELECT SUBSTRING("first_name",1,2) modified, COUNT(*) count FROM "test_emp" WHERE SUBSTRING("first_name",1,2)='Al' GROUP BY SUBSTRING("first_name",1,2) LIMIT 10;
+
+modified:s  | count:l
+Al          | 3
+;
+
+//this one doesn't return anything. The problem is "IS NOT NULL". GH issue created to generally investigate the NULLs usage: https://github.com/elastic/elasticsearch/issues/32079
+//selectSubstringWithWhereNotNullAndCountGroupBy
+//SELECT SUBSTRING("first_name",5,20) modified, COUNT(*) count FROM "test_emp" WHERE SUBSTRING("first_name",5,20) IS NOT NULL GROUP BY SUBSTRING("first_name",5,20) ORDER BY SUBSTRING("first_name",5,20) LIMIT 10;
+
+//   modified:s  |     count:l
+//---------------+---------------
+//               |15
+//adram          |1
+//af             |1
+//aja            |1
+//al             |1
+//andro          |2
+//antios         |1
+//ard            |1
+//areta          |1
+//arsan          |1
+//;
+
+selectSubstringWithWhereAndGroupBy
+SELECT SUBSTRING("first_name",5,20) modified, COUNT(*) count FROM "test_emp" GROUP BY SUBSTRING("first_name",5,20) ORDER BY SUBSTRING("first_name",5,20) LIMIT 10;
+
+   modified:s  |     count:l
+---------------+---------------
+               |15
+adram          |1
+af             |1
+aja            |1
+al             |1
+andro          |2
+antios         |1
+ard            |1
+areta          |1
+arsan          |1
+;
+
+selectReplace
+SELECT REPLACE("first_name",'A','X') modified, "first_name" origFN FROM "test_emp" ORDER BY "first_name" LIMIT 10;
+
+   modified:s  |    origFN:s
+---------------+---------------
+Xdamantios     |Adamantios
+Xlain          |Alain
+Xlejandro      |Alejandro
+Xlejandro      |Alejandro
+Xmabile        |Amabile
+Xnneke         |Anneke
+Xnoosh         |Anoosh
+Xrif           |Arif
+Xrumugam       |Arumugam
+Bader          |Bader
+;
+
+selectReplaceWithGroupBy
+SELECT REPLACE("first_name",'jan','_JAN_') modified, COUNT(*) count FROM "test_emp" GROUP BY REPLACE("first_name",'jan','_JAN_') ORDER BY REPLACE("first_name",'jan','_JAN_') LIMIT 10;
+
+   modified:s  |     count:l
+---------------+---------------
+Adamantios     |1
+Alain          |1
+Ale_JAN_dro    |2
+Amabile        |1
+Anneke         |1
+Anoosh         |1
+Arif           |1
+Arumugam       |1
+Bader          |1
+Basil          |1
+;
+
+selectReplaceWithCastAndCondition
+SELECT REPLACE(CAST("languages" AS VARCHAR),'1','100') foo, "languages" FROM "test_emp" WHERE "languages"=1 OR "languages"=2 LIMIT 5;
+
+      foo:s    |   languages:bt
+---------------+---------------
+2              |2
+100            |1
+2              |2
+100            |1
+100            |1
+;
+
+selectPositionWithConditionAndLcase
+SELECT POSITION('x',LCASE("first_name")) pos, "first_name" FROM "test_emp" WHERE POSITION('x',LCASE("first_name")) != 0;
+
+      pos:i    |  first_name:s
+---------------+---------------
+4              |Guoxiang
+1              |Xinglin
+;
+
+selectPositionWithLcaseAndConditionWithGroupByAndOrderBy
+SELECT POSITION('m',LCASE("first_name")), COUNT(*) pos FROM "test_emp" WHERE POSITION('m',LCASE("first_name")) != 0 GROUP BY POSITION('m',LCASE("first_name")) ORDER BY POSITION('m',LCASE("first_name")) DESC;
+
+POSITION(m,LCASE(first_name)):i|      pos:l
+-------------------------------+---------------
+9                              |1
+7                              |1
+4                              |3
+3                              |6
+2                              |1
+1                              |9
+;
+
+selectInsertWithPositionAndCondition
+SELECT INSERT("first_name",POSITION('m',"first_name"),1,'M') modified, POSITION('m',"first_name") pos FROM "test_emp" WHERE POSITION('m',"first_name") > 1;
+
+ modified:s    |      pos:i
+---------------+---------------
+SuMant         |3
+RaMzi          |3
+PrasadraM      |9
+DoMenick       |3
+OtMar          |3
+AdaMantios     |4
+HidefuMi       |7
+MayuMi         |5
+PreMal         |4
+SoMnath        |3
+AMabile        |2
+AruMugam       |4
+ReMzi          |3
+;
+
+selectLocateAndInsertWithLocateWithConditionAndThreeParameters
+SELECT LOCATE('a',"first_name",7) pos, INSERT("first_name",LOCATE('a',"first_name",7),1,'AAA') FROM "test_emp" WHERE LOCATE('a',"first_name",7) > 0;
+
+    pos:i      |INSERT(first_name,LOCATE(a,first_name,7),1,AAA):s
+---------------+-----------------------------------------------
+8              |ChirstiAAAn
+7              |DuangkAAAew
+8              |PrasadrAAAm
+7              |YongqiAAAo
+7              |YinghuAAA
+8              |BreanndAAA
+9              |MargaretAAA
+8              |SudharsAAAn
+7              |SailajAAA
+7              |ArumugAAAm
+11             |SreekrishnAAA
+;
+
+selectLocateAndInsertWithLocateWithConditionAndTwoParameters
+SELECT LOCATE('a',"first_name") pos, INSERT("first_name",LOCATE('a',"first_name"),1,'AAA') FROM "test_emp" WHERE LOCATE('a',"first_name") > 0 ORDER BY "first_name" LIMIT 10;
+
+    pos:i      |INSERT(first_name,LOCATE(a,first_name),1,AAA):s
+---------------+---------------------------------------------
+3              |AdAAAmantios
+3              |AlAAAin
+5              |AlejAAAndro
+5              |AlejAAAndro
+3              |AmAAAbile
+7              |ArumugAAAm
+2              |BAAAder
+2              |BAAAsil
+5              |BerhAAArd
+4              |BezAAAlel
+;
+
+selectLeft
+SELECT LEFT("first_name",2) f FROM "test_emp" ORDER BY "first_name" LIMIT 10;
+
+f:s
+Ad
+Al
+Al
+Al
+Am
+An
+An
+Ar
+Ar
+Ba
+;
+
+selectRight
+SELECT RIGHT("first_name",2) f FROM "test_emp" ORDER BY "first_name" LIMIT 10;
+
+f:s
+os
+in
+ro
+ro
+le
+ke
+sh
+if
+am
+er
+;
+
+selectRightWithGroupByAndOrderBy
+SELECT RIGHT("first_name",2) f, COUNT(*) count FROM "test_emp" GROUP BY RIGHT("first_name",2) ORDER BY RIGHT("first_name",2) LIMIT 10;
+
+    f:s        |count:l
+---------------+---------------------------------------------
+af             |1
+al             |2
+am             |2
+an             |7
+ao             |3
+ar             |2
+ay             |1
+be             |1
+bu             |1
+by             |1
+;
+
+upperCasingTheSecondLetterFromTheRightFromFirstName
+SELECT CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) f FROM "test_emp" ORDER BY "first_name" LIMIT 10;
+
+
+f:s 
+AdamantiOs
+AlaIn
+AlejandRo
+AlejandRo
+AmabiLe
+AnneKe
+AnooSh
+ArIf
+ArumugAm
+BadEr
+;
+
+upperCasingTheSecondLetterFromTheRightFromFirstNameWithOrderByAndGroupBy
+SELECT CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) f, COUNT(*) c FROM "test_emp" GROUP BY CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) ORDER BY CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) LIMIT 10;
+
+    f:s        |c:l
+---------------+---------------------------------------------
+AdamantiOs     |1
+AlaIn          |1
+AlejandRo      |2
+AmabiLe        |1
+AnneKe         |1
+AnooSh         |1
+ArIf           |1
+ArumugAm       |1
+BadEr          |1
+BasIl          |1
+;
+
+upperCasingTheSecondLetterFromTheRightFromFirstNameWithWhere
+SELECT CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) f, COUNT(*) c FROM "test_emp" WHERE CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1))='AlejandRo' GROUP BY CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) ORDER BY CONCAT(CONCAT(SUBSTRING("first_name",1,LENGTH("first_name")-2),UCASE(LEFT(RIGHT("first_name",2),1))),RIGHT("first_name",1)) LIMIT 10;
+
+    f:s        |c:l
+---------------+---------------------------------------------
+AlejandRo      |2
+;
diff --git a/x-pack/qa/sql/src/main/resources/string-functions.sql-spec b/x-pack/qa/sql/src/main/resources/string-functions.sql-spec
index d9a35edf1b030..15bb6dea935c8 100644
--- a/x-pack/qa/sql/src/main/resources/string-functions.sql-spec
+++ b/x-pack/qa/sql/src/main/resources/string-functions.sql-spec
@@ -22,7 +22,8 @@ SELECT LCASE(first_name) lc, CHAR(ASCII(LCASE(first_name))) chr FROM "test_emp"
 ltrimFilter
 SELECT LTRIM(first_name) lt FROM "test_emp" WHERE LTRIM(first_name) = 'Bob';
 
-//Unsupported yet
+// Unsupported yet
+// Functions combined with 'LIKE' should perform the match inside a Painless script, whereas at the moment it's handled as a regular `match` query in ES.
 //ltrimFilterWithLike
 //SELECT LTRIM("first_name") lt FROM "test_emp" WHERE LTRIM("first_name") LIKE '%a%';
 
@@ -74,3 +75,66 @@ SELECT SPACE("languages") s, COUNT(*) count FROM "test_emp" GROUP BY SPACE("lang
 
 spaceGroupByAndOrderByWithCharLength
 SELECT SPACE("languages") s, COUNT(*) count, CAST(CHAR_LENGTH(SPACE("languages")) AS INT) cls FROM "test_emp" WHERE "languages" IS NOT NULL GROUP BY SPACE("languages") ORDER BY SPACE("languages");
+
+selectConcatWithOrderBy
+SELECT first_name f, last_name l, CONCAT(first_name,last_name) cct FROM test_emp ORDER BY CONCAT(first_name,last_name) LIMIT 10;
+
+selectNestedConcatWithOrderBy
+SELECT first_name f, last_name l, CONCAT(first_name,CONCAT(' ',last_name)) cct FROM test_emp ORDER BY CONCAT(first_name,CONCAT(' ',last_name)) LIMIT 10;
+
+selectConcatWithGroupBy
+SELECT CONCAT(first_name,last_name) cct FROM test_emp GROUP BY CONCAT(first_name,last_name) ORDER BY CONCAT(first_name,last_name) LIMIT 1;
+
+selectAsciiOfConcatWithGroupByOrderByCount
+SELECT ASCII(CONCAT("first_name","last_name")) ascii, COUNT(*) count FROM "test_emp" GROUP BY ASCII(CONCAT("first_name","last_name")) ORDER BY ASCII(CONCAT("first_name","last_name")) DESC LIMIT 10;
+
+selectRepeatTwice
+SELECT "first_name" orig, REPEAT("first_name",2) reps FROM "test_emp" WHERE ASCII("first_name")=65 ORDER BY REPEAT("first_name",2) ASC LIMIT 10;
+
+selectInsertWithLcase
+SELECT "first_name" orig, INSERT("first_name",2,1000,LCASE("first_name")) modified FROM "test_emp" WHERE ASCII("first_name")=65 ORDER BY "first_name" ASC LIMIT 10;
+
+selectInsertWithUcaseWithGroupByAndOrderBy
+SELECT INSERT(UCASE("first_name"),2,123000,INSERT(UCASE("last_name"),2,500,' ')) modified, COUNT(*) count FROM "test_emp" WHERE ASCII("first_name")=65 GROUP BY INSERT(UCASE("first_name"),2,123000,INSERT(UCASE("last_name"),2,500,' ')) ORDER BY INSERT(UCASE("first_name"),2,123000,INSERT(UCASE("last_name"),2,500,' ')) ASC LIMIT 10;
+
+selectSubstringWithGroupBy
+SELECT SUBSTRING("first_name",1,2) modified, COUNT(*) count FROM "test_emp" WHERE ASCII("first_name")=65 GROUP BY SUBSTRING("first_name",1,2) ORDER BY SUBSTRING("first_name",1,2) ASC LIMIT 10;
+
+selectSubstringWithWhereCountAndGroupBy
+SELECT SUBSTRING("first_name",1,2) modified, COUNT(*) count FROM "test_emp" WHERE SUBSTRING("first_name",1,2)='Al' GROUP BY SUBSTRING("first_name",1,2) LIMIT 10;
+
+selectSubstringWithWhereAndGroupBy
+SELECT SUBSTRING("first_name",5,20) modified, COUNT(*) count FROM "test_emp" GROUP BY SUBSTRING("first_name",5,20) ORDER BY SUBSTRING("first_name",5,20) LIMIT 10;
+
+selectReplace
+SELECT REPLACE("first_name",'A','X') modified, "first_name" origFN FROM "test_emp" ORDER BY "first_name" LIMIT 10;
+
+selectReplaceWithGroupBy
+SELECT REPLACE("first_name",'jan','_JAN_') modified, COUNT(*) count FROM "test_emp" GROUP BY REPLACE("first_name",'jan','_JAN_') ORDER BY REPLACE("first_name",'jan','_JAN_') LIMIT 10;
+
+selectReplaceWithCastAndCondition
+SELECT REPLACE(CAST("languages" AS VARCHAR),'1','100') foo, "languages" FROM "test_emp" WHERE "languages"=1 OR "languages"=2 LIMIT 5;
+
+selectPositionWithConditionAndLcase
+SELECT POSITION('x',LCASE("first_name")) pos, "first_name" FROM "test_emp" WHERE POSITION('x',LCASE("first_name")) != 0;
+
+selectPositionWithLcaseAndConditionWithGroupByAndOrderBy
+SELECT POSITION('m',LCASE("first_name")) posOfM, COUNT(*) pos FROM "test_emp" WHERE POSITION('m',LCASE("first_name")) != 0 GROUP BY POSITION('m',LCASE("first_name")) ORDER BY POSITION('m',LCASE("first_name")) DESC;
+
+selectInsertWithPositionAndCondition
+SELECT INSERT("first_name",POSITION('m',"first_name"),1,'M') modified, POSITION('m',"first_name") pos FROM "test_emp" WHERE POSITION('m',"first_name") > 1;
+
+selectLocateAndInsertWithLocateWithConditionAndThreeParameters
+SELECT LOCATE('a',"first_name",7) pos, INSERT("first_name",LOCATE('a',"first_name",7),1,'AAA') inserted FROM "test_emp" WHERE LOCATE('a',"first_name",7) > 0;
+
+selectLocateAndInsertWithLocateWithConditionAndTwoParameters
+SELECT LOCATE('a',"first_name") pos, INSERT("first_name",LOCATE('a',"first_name"),1,'AAA') inserted FROM "test_emp" WHERE LOCATE('a',"first_name") > 0 ORDER BY "first_name" LIMIT 10;
+
+selectLeft
+SELECT LEFT("first_name",2) f FROM "test_emp" ORDER BY "first_name" LIMIT 10;
+
+selectRight
+SELECT RIGHT("first_name",2) f FROM "test_emp" ORDER BY "first_name" LIMIT 10;
+
+selectRightWithGroupByAndOrderBy
+SELECT RIGHT("first_name",2) f, COUNT(*) count FROM "test_emp" GROUP BY RIGHT("first_name",2) ORDER BY RIGHT("first_name",2) LIMIT 10;